Loading ...
Sorry, an error occurred while loading the content.
 

Re: conf_maxdumpsize should be unsigned long int

Expand Messages
  • Sven Rudolph
    ... I learned that long long is the needed type for a 64bit conf_maxdumpsize ... The modifcations below gave me a planner that seems to do the right thing. For
    Message 1 of 3 , Oct 4, 2005
      Sven Rudolph <Sven_Rudolph@...> writes:

      > In amanda.conf I set:
      >
      > maxdumpsize 10000 GB
      >
      > (I want it to be "unlimited", so I chose "a very large number". This
      > amanda instance only does full backups, so most of the planner magic
      > isn't needed. Here it is not planners job to make something fail.)
      >
      >
      > I added some new DLEs. Now some dumps failed due to
      >
      > planner: [dumps too big, xxx KB, but no incremental estimate]
      >
      >
      > /var/log/amanda/WeeklySet1/amdump says:
      >
      > DELAYING DUMPS IF NEEDED, total_size 1978410666, tape length 1895825408 mark 0
      >
      > This is surprising, since I tould it that the limit is 10000 GB, not
      > 1895825408 bytes. So I asked bc:
      >
      > 1895825408 / 1024 / 1024
      > 1808
      > 1808 + 4096 + 4096
      > 10000
      >
      > Obviously the bits above the 32nd were chopped off.
      >
      > In server-src/planner.c the variable conf_maxdumpsize is defined as
      > int. It should be defined as "unsigned long int" (as is tape_length).
      > (A getconf_longint might be needed too.)

      I learned that long long is the needed type for a 64bit
      conf_maxdumpsize ...

      The modifcations below gave me a planner that seems to do the right
      thing. For testing I only ran planner, not the complete dump. The real
      amanda run will happen on weekend.

      My change of l in val_t might be considered to be unclean; the
      alternative approach were an extra field ll and an type token LONGLONG
      in tok_t, but I didn't get this running.

      Sven

      -----

      *** amanda-2.4.5.original/server-src/conffile.c Tue Mar 29 18:35:11 2005
      --- amanda-2.4.5/server-src/conffile.c Tue Oct 4 14:30:09 2005
      ***************
      *** 135,141 ****

      typedef union {
      int i;
      ! long l;
      double r;
      char *s;
      } val_t;
      --- 135,141 ----

      typedef union {
      int i;
      ! long long l;
      double r;
      char *s;
      } val_t;
      ***************
      *** 322,328 ****

      static void get_simple P((val_t *var, int *seen, tok_t type));
      static int get_time P((void));
      ! static long get_number P((void));
      static int get_bool P((void));
      static void ckseen P((int *seen));
      static void parserror P((char *format, ...))
      --- 322,328 ----

      static void get_simple P((val_t *var, int *seen, tok_t type));
      static int get_time P((void));
      ! static long long get_number P((void));
      static int get_bool P((void));
      static void ckseen P((int *seen));
      static void parserror P((char *format, ...))
      ***************
      *** 425,431 ****
      { "DISPLAYUNIT", CNF_DISPLAYUNIT, STRING },
      { "AUTOFLUSH", CNF_AUTOFLUSH, BOOL },
      { "RESERVE", CNF_RESERVE, INT },
      ! { "MAXDUMPSIZE", CNF_MAXDUMPSIZE, INT },
      { NULL }
      };

      --- 425,431 ----
      { "DISPLAYUNIT", CNF_DISPLAYUNIT, STRING },
      { "AUTOFLUSH", CNF_AUTOFLUSH, BOOL },
      { "RESERVE", CNF_RESERVE, INT },
      ! { "MAXDUMPSIZE", CNF_MAXDUMPSIZE, LONG},
      { NULL }
      };

      ***************
      *** 547,553 ****
      case CNF_TAPEBUFS: r = conf_tapebufs.i; break;
      case CNF_AUTOFLUSH: r = conf_autoflush.i; break;
      case CNF_RESERVE: r = conf_reserve.i; break;
      - case CNF_MAXDUMPSIZE: r = conf_maxdumpsize.i; break;
      case CNF_AMRECOVER_DO_FSF: r = conf_amrecover_do_fsf.i; break;
      case CNF_AMRECOVER_CHECK_LABEL: r = conf_amrecover_check_label.i; break;
      case CNF_TAPERALGO: r = conf_taperalgo.i; break;
      --- 547,552 ----
      ***************
      *** 559,564 ****
      --- 558,578 ----
      return r;
      }

      + long long getconf_longlong(parm)
      + confparm_t parm;
      + {
      + long long r = 0;
      +
      + switch(parm) {
      + case CNF_MAXDUMPSIZE: r = conf_maxdumpsize.l; break;
      +
      + default:
      + error("error [unknown getconf_int parm: %d]", parm);
      + /* NOTREACHED */
      + }
      + return r;
      + }
      +
      double getconf_real(parm)
      confparm_t parm;
      {
      ***************
      *** 744,750 ****
      conf_tapebufs.i = 20;
      conf_autoflush.i = 0;
      conf_reserve.i = 100;
      ! conf_maxdumpsize.i = -1;
      conf_amrecover_do_fsf.i = 0;
      conf_amrecover_check_label.i = 0;
      conf_taperalgo.i = 0;
      --- 758,764 ----
      conf_tapebufs.i = 20;
      conf_autoflush.i = 0;
      conf_reserve.i = 100;
      ! conf_maxdumpsize.l = -1;
      conf_amrecover_do_fsf.i = 0;
      conf_amrecover_check_label.i = 0;
      conf_taperalgo.i = 0;
      ***************
      *** 1093,1099 ****
      parserror("reserve must be between 0 and 100");
      }
      break;
      ! case MAXDUMPSIZE:get_simple(&conf_maxdumpsize,&seen_maxdumpsize,INT); break;
      case COLUMNSPEC:get_simple(&conf_columnspec,&seen_columnspec,STRING); break;

      case AMRECOVER_DO_FSF: get_simple(&conf_amrecover_do_fsf,&seen_amrecover_do_fsf, BOOL); break;
      --- 1107,1113 ----
      parserror("reserve must be between 0 and 100");
      }
      break;
      ! case MAXDUMPSIZE:get_simple(&conf_maxdumpsize,&seen_maxdumpsize,LONG); break;
      case COLUMNSPEC:get_simple(&conf_columnspec,&seen_columnspec,STRING); break;

      case AMRECOVER_DO_FSF: get_simple(&conf_amrecover_do_fsf,&seen_amrecover_do_fsf, BOOL); break;
      ***************
      *** 2552,2560 ****
      { NULL, IDENT }
      };

      ! static long get_number()
      {
      ! long val;
      keytab_t *save_kt;

      save_kt = keytable;
      --- 2566,2574 ----
      { NULL, IDENT }
      };

      ! static long long get_number()
      {
      ! long long val;
      keytab_t *save_kt;

      save_kt = keytable;
      ***************
      *** 2564,2573 ****

      switch(tok) {
      case INT:
      ! val = (long) tokenval.i;
      break;
      case INFINITY:
      ! val = (long) BIGINT;
      break;
      default:
      parserror("an integer expected");
      --- 2578,2587 ----

      switch(tok) {
      case INT:
      ! val = (long long) tokenval.i;
      break;
      case INFINITY:
      ! val = (long long) BIGINT;
      break;
      default:
      parserror("an integer expected");
      diff -c -r amanda-2.4.5.original/server-src/conffile.h amanda-2.4.5/server-src/conffile.h
      *** amanda-2.4.5.original/server-src/conffile.h Tue Mar 29 18:35:11 2005
      --- amanda-2.4.5/server-src/conffile.h Tue Oct 4 14:01:53 2005
      ***************
      *** 282,287 ****
      --- 282,288 ----
      int read_conffile P((char *filename));
      int getconf_seen P((confparm_t parameter));
      int getconf_int P((confparm_t parameter));
      + long long getconf_longlong P((confparm_t parameter));
      double getconf_real P((confparm_t parameter));
      char *getconf_str P((confparm_t parameter));
      char *getconf_byname P((char *confname));
      *** amanda-2.4.5.original/server-src/planner.c Wed Mar 16 19:09:50 2005
      --- amanda-2.4.5/server-src/planner.c Tue Oct 4 14:04:10 2005
      ***************
      *** 53,59 ****
      /* configuration file stuff */

      char *conf_tapetype;
      ! int conf_maxdumpsize;
      int conf_runtapes;
      int conf_dumpcycle;
      int conf_runspercycle;
      --- 53,59 ----
      /* configuration file stuff */

      char *conf_tapetype;
      ! long long conf_maxdumpsize;
      int conf_runtapes;
      int conf_dumpcycle;
      int conf_runspercycle;
      ***************
      *** 98,104 ****
      disklist_t startq, waitq, pestq, estq, failq, schedq;
      long total_size;
      double total_lev0, balanced_size, balance_threshold;
      ! unsigned long tape_length, tape_mark;
      int result_port, amanda_port;

      static am_feature_t *our_features = NULL;
      --- 98,105 ----
      disklist_t startq, waitq, pestq, estq, failq, schedq;
      long total_size;
      double total_lev0, balanced_size, balance_threshold;
      ! unsigned long long tape_length;
      ! unsigned long tape_mark;
      int result_port, amanda_port;

      static am_feature_t *our_features = NULL;
      ***************
      *** 301,307 ****
      amfree(conf_infofile);

      conf_tapetype = getconf_str(CNF_TAPETYPE);
      ! conf_maxdumpsize = getconf_int(CNF_MAXDUMPSIZE);
      conf_runtapes = getconf_int(CNF_RUNTAPES);
      conf_dumpcycle = getconf_int(CNF_DUMPCYCLE);
      conf_runspercycle = getconf_int(CNF_RUNSPERCYCLE);
      --- 302,308 ----
      amfree(conf_infofile);

      conf_tapetype = getconf_str(CNF_TAPETYPE);
      ! conf_maxdumpsize = getconf_longlong(CNF_MAXDUMPSIZE);
      conf_runtapes = getconf_int(CNF_RUNTAPES);
      conf_dumpcycle = getconf_int(CNF_DUMPCYCLE);
      conf_runspercycle = getconf_int(CNF_RUNSPERCYCLE);
      ***************
      *** 508,514 ****
      */

      fprintf(stderr,
      ! "\nDELAYING DUMPS IF NEEDED, total_size %ld, tape length %lu mark %lu\n",
      total_size, tape_length, tape_mark);

      initial_size = total_size;
      --- 509,515 ----
      */

      fprintf(stderr,
      ! "\nDELAYING DUMPS IF NEEDED, total_size %ld, tape length %llu mark %lu\n",
      total_size, tape_length, tape_mark);

      initial_size = total_size;
    • Sven Rudolph
      Some weeks ago (see References: header) I tried to increase the potential size of conf_maxdumpsize so that it can represent more than 2 Terabyte. From your
      Message 2 of 3 , Nov 7, 2005
        Some weeks ago (see References: header) I tried to increase the
        potential size of conf_maxdumpsize so that it can represent more than
        2 Terabyte. From your discussion I learned that going 64 bit is
        troublesome. (And I did some reading in a C book ...)

        (Right now I don't use the patch I posted, instead I set tapelength
        and runtapes so that the product is a bit below 4TB.)

        Due to continuous data growth the total dump size is now above 2TB,
        and a problem showed up in another place. When the planner algorithm
        runs, totalsize (the sum of all the dump sizes) is stored in a signed
        long.

        In order to avoid the trouble with "long long" I instead tried to
        change the relevant variables to unsigned.

        On testing: the changed amanda worked the last two weekends. Patch
        appended below.

        Sven

        -----
        diff -c -r amanda-2.4.5.original/server-src/planner.c amanda-2.4.5/server-src/planner.c
        *** amanda-2.4.5.original/server-src/planner.c Wed Mar 16 19:09:50 2005
        --- amanda-2.4.5/server-src/planner.c Wed Oct 26 13:36:29 2005
        ***************
        *** 96,104 ****

        /* pestq = partial estimate */
        disklist_t startq, waitq, pestq, estq, failq, schedq;
        ! long total_size;
        double total_lev0, balanced_size, balance_threshold;
        ! unsigned long tape_length, tape_mark;
        int result_port, amanda_port;

        static am_feature_t *our_features = NULL;
        --- 96,105 ----

        /* pestq = partial estimate */
        disklist_t startq, waitq, pestq, estq, failq, schedq;
        ! unsigned long total_size;
        double total_lev0, balanced_size, balance_threshold;
        ! unsigned long tape_length;
        ! unsigned long tape_mark;
        int result_port, amanda_port;

        static am_feature_t *our_features = NULL;
        ***************
        *** 127,133 ****
        int deleted; /* 0=modified, 1=deleted */
        disk_t *dp; /* The disk that was changed */
        int level; /* The original level */
        ! long size; /* The original size */
        char *errstr; /* A message describing why this disk is here */
        } bi_t;

        --- 128,134 ----
        int deleted; /* 0=modified, 1=deleted */
        disk_t *dp; /* The disk that was changed */
        int level; /* The original level */
        ! unsigned long size; /* The original size */
        char *errstr; /* A message describing why this disk is here */
        } bi_t;

        ***************
        *** 164,170 ****
        char **vp;
        unsigned long malloc_hist_1, malloc_size_1;
        unsigned long malloc_hist_2, malloc_size_2;
        ! long initial_size;
        int fd;
        char *conffile;
        char *conf_diskfile;
        --- 165,171 ----
        char **vp;
        unsigned long malloc_hist_1, malloc_size_1;
        unsigned long malloc_hist_2, malloc_size_2;
        ! unsigned long initial_size;
        int fd;
        char *conffile;
        char *conf_diskfile;
        ***************
        *** 341,347 ****
        tape_length = conf_maxdumpsize;
        }
        else {
        ! tape_length = tape->length * conf_runtapes;
        }
        tape_mark = tape->filemark;
        tt_blocksize_kb = tape->blocksize;
        --- 342,349 ----
        tape_length = conf_maxdumpsize;
        }
        else {
        ! tape_length = (unsigned long) tape->length
        ! * (unsigned long) conf_runtapes;
        }
        tape_mark = tape->filemark;
        tt_blocksize_kb = tape->blocksize;
        ***************
        *** 486,492 ****
        {
        disk_t *dp;

        ! fprintf(stderr, "INITIAL SCHEDULE (size %ld):\n", total_size);
        for(dp = schedq.head; dp != NULL; dp = dp->next) {
        fprintf(stderr, " %s %s pri %d lev %d size %ld\n",
        dp->host->hostname, dp->name, est(dp)->dump_priority,
        --- 488,494 ----
        {
        disk_t *dp;

        ! fprintf(stderr, "INITIAL SCHEDULE (size %lu):\n", total_size);
        for(dp = schedq.head; dp != NULL; dp = dp->next) {
        fprintf(stderr, " %s %s pri %d lev %d size %ld\n",
        dp->host->hostname, dp->name, est(dp)->dump_priority,
        ***************
        *** 508,514 ****
        */

        fprintf(stderr,
        ! "\nDELAYING DUMPS IF NEEDED, total_size %ld, tape length %lu mark %lu\n",
        total_size, tape_length, tape_mark);

        initial_size = total_size;
        --- 510,516 ----
        */

        fprintf(stderr,
        ! "\nDELAYING DUMPS IF NEEDED, total_size %lu, tape length %lu mark %lu\n",
        total_size, tape_length, tape_mark);

        initial_size = total_size;
        ***************
        *** 1956,1962 ****
        balanced_size += lev0size / runs_per_cycle;
        }

        ! fprintf(stderr,"total size %ld total_lev0 %1.0f balanced-lev0size %1.0f\n",
        total_size, total_lev0, balanced_size);
        }

        --- 1958,1964 ----
        balanced_size += lev0size / runs_per_cycle;
        }

        ! fprintf(stderr,"total size %lu total_lev0 %1.0f balanced-lev0size %1.0f\n",
        total_size, total_lev0, balanced_size);
        }

        ***************
        *** 2116,2122 ****
        {
        disk_t *dp, *ndp, *preserve;
        bi_t *bi, *nbi;
        ! long new_total; /* New total_size */
        char est_kb[20]; /* Text formatted dump size */
        int nb_forced_level_0;
        info_t info;
        --- 2118,2124 ----
        {
        disk_t *dp, *ndp, *preserve;
        bi_t *bi, *nbi;
        ! unsigned long new_total; /* New total_size */
        char est_kb[20]; /* Text formatted dump size */
        int nb_forced_level_0;
        info_t info;
        ***************
        *** 2365,2371 ****
        amfree(bi);
        }

        ! fprintf(stderr, " delay: Total size now %ld.\n", total_size);

        return;
        }
        --- 2367,2373 ----
        amfree(bi);
        }

        ! fprintf(stderr, " delay: Total size now %lu.\n", total_size);

        return;
        }
        ***************
        *** 2435,2441 ****
        static int promote_highest_priority_incremental P((void))
        {
        disk_t *dp, *dp1, *dp_promote;
        ! long new_size, new_total, new_lev0;
        int check_days;
        int nb_today, nb_same_day, nb_today2;
        int nb_disk_today, nb_disk_same_day;
        --- 2437,2443 ----
        static int promote_highest_priority_incremental P((void))
        {
        disk_t *dp, *dp1, *dp_promote;
        ! unsigned long new_size, new_total, new_lev0;
        int check_days;
        int nb_today, nb_same_day, nb_today2;
        int nb_disk_today, nb_disk_same_day;
        ***************
        *** 2531,2537 ****
        est(dp)->next_level0 = 0;

        fprintf(stderr,
        ! " promote: moving %s:%s up, total_lev0 %1.0f, total_size %ld\n",
        dp->host->hostname, dp->name,
        total_lev0, total_size);

        --- 2533,2539 ----
        est(dp)->next_level0 = 0;

        fprintf(stderr,
        ! " promote: moving %s:%s up, total_lev0 %1.0f, total_size %lu\n",
        dp->host->hostname, dp->name,
        total_lev0, total_size);

        ***************
        *** 2555,2562 ****
        int days;
        int hill_days = 0;
        long hill_size;
        ! long new_size;
        ! long new_total;
        int my_dumpcycle;

        /* If we are already doing a level 0 don't bother */
        --- 2557,2564 ----
        int days;
        int hill_days = 0;
        long hill_size;
        ! unsigned long new_size;
        ! unsigned long new_total;
        int my_dumpcycle;

        /* If we are already doing a level 0 don't bother */
        ***************
        *** 2617,2623 ****
        est(dp)->dump_size = new_size;

        fprintf(stderr,
        ! " promote: moving %s:%s up, total_lev0 %1.0f, total_size %ld\n",
        dp->host->hostname, dp->name,
        total_lev0, total_size);

        --- 2619,2625 ----
        est(dp)->dump_size = new_size;

        fprintf(stderr,
        ! " promote: moving %s:%s up, total_lev0 %1.0f, total_size %lu\n",
        dp->host->hostname, dp->name,
        total_lev0, total_size);
      Your message has been successfully submitted and would be delivered to recipients shortly.