Line data Source code
1 : #include "../../burp.h"
2 : #include "../../alloc.h"
3 : #include "../../conf.h"
4 : #include "../../conffile.h"
5 : #include "../../handy.h"
6 : #include "../../fsops.h"
7 : #include "../../fzp.h"
8 : #include "../../lock.h"
9 : #include "../../log.h"
10 : #include "../../prepend.h"
11 : #include "../../strlist.h"
12 : #include "bedup.h"
13 :
14 : #include <uthash.h>
15 :
16 : #define LOCKFILE_NAME "lockfile"
17 : #define BEDUP_LOCKFILE_NAME "lockfile.bedup"
18 :
19 : #define DEF_MAX_LINKS 10000
20 :
21 : static int makelinks=0;
22 : static int deletedups=0;
23 :
24 : static uint64_t savedbytes=0;
25 : static uint64_t count=0;
26 : static int ccount=0;
27 :
28 : static struct lock *locklist=NULL;
29 :
30 : static int verbose=0;
31 :
32 : static unsigned int maxlinks=DEF_MAX_LINKS;
33 : static char ext[16]="";
34 :
35 : typedef struct file file_t;
36 :
37 : struct file
38 : {
39 : char *path;
40 : dev_t dev;
41 : ino_t ino;
42 : nlink_t nlink;
43 : uint64_t full_cksum;
44 : uint64_t part_cksum;
45 : file_t *next;
46 : };
47 :
48 : struct mystruct
49 : {
50 : off_t st_size;
51 : file_t *files;
52 : UT_hash_handle hh;
53 : };
54 :
55 : struct mystruct *myfiles=NULL;
56 :
57 4 : static struct mystruct *find_key(off_t st_size)
58 : {
59 : struct mystruct *s;
60 :
61 4 : HASH_FIND_INT(myfiles, &st_size, s);
62 4 : return s;
63 : }
64 :
65 2 : static int add_file(struct mystruct *s, struct file *f)
66 : {
67 : struct file *newfile;
68 2 : if(!(newfile=(struct file *)malloc_w(sizeof(struct file), __func__)))
69 : return -1;
70 : memcpy(newfile, f, sizeof(struct file));
71 2 : f->path=NULL;
72 2 : newfile->next=s->files;
73 2 : s->files=newfile;
74 : return 0;
75 : }
76 :
77 2 : static int add_key(off_t st_size, struct file *f)
78 : {
79 : struct mystruct *s;
80 :
81 2 : if(!(s=(struct mystruct *)malloc_w(sizeof(struct mystruct), __func__)))
82 : return -1;
83 2 : s->st_size=st_size;
84 2 : s->files=NULL;
85 2 : if(add_file(s, f)) return -1;
86 : //printf("HASH ADD %d\n", st_size);
87 6 : HASH_ADD_INT(myfiles, st_size, s);
88 : return 0;
89 : }
90 :
91 : static void file_free_content(struct file *file)
92 : {
93 2 : if(!file) return;
94 2 : free_w(&file->path);
95 : }
96 :
97 2 : static void file_free(struct file **file)
98 : {
99 4 : if(!file || !*file) return;
100 2 : file_free_content(*file);
101 2 : free_v((void **)file);
102 : }
103 :
104 2 : static void files_free(struct file **files)
105 : {
106 : struct file *f;
107 : struct file *fhead;
108 2 : if(!files || !*files) return;
109 : fhead=*files;
110 4 : while(fhead)
111 : {
112 2 : f=fhead;
113 2 : fhead=fhead->next;
114 2 : file_free(&f);
115 : }
116 : }
117 :
118 : static void mystruct_free_content(struct mystruct *mystruct)
119 : {
120 2 : if(!mystruct) return;
121 2 : files_free(&mystruct->files);
122 : }
123 :
124 2 : static void mystruct_free(struct mystruct **mystruct)
125 : {
126 4 : if(!mystruct || !*mystruct) return;
127 2 : mystruct_free_content(*mystruct);
128 2 : free_v((void **)mystruct);
129 : }
130 :
131 2 : static void mystruct_delete_all(void)
132 : {
133 : struct mystruct *tmp;
134 : struct mystruct *mystruct;
135 :
136 4 : HASH_ITER(hh, myfiles, mystruct, tmp)
137 : {
138 2 : HASH_DEL(myfiles, mystruct);
139 2 : mystruct_free(&mystruct);
140 : }
141 2 : myfiles=NULL;
142 2 : }
143 :
144 : #define FULL_CHUNK 4096
145 :
146 2 : static int full_match(struct file *o, struct file *n,
147 : struct fzp **ofp, struct fzp **nfp)
148 : {
149 : size_t ogot;
150 : size_t ngot;
151 2 : unsigned int i=0;
152 : static char obuf[FULL_CHUNK];
153 : static char nbuf[FULL_CHUNK];
154 :
155 2 : if(*ofp) fzp_seek(*ofp, 0, SEEK_SET);
156 0 : else if(!(*ofp=fzp_open(o->path, "rb")))
157 : {
158 : // Blank this entry so that it can be ignored from
159 : // now on.
160 0 : free_w(&o->path);
161 : return 0;
162 : }
163 :
164 2 : if(*nfp) fzp_seek(*nfp, 0, SEEK_SET);
165 0 : else if(!(*nfp=fzp_open(n->path, "rb"))) return 0;
166 :
167 : while(1)
168 : {
169 2 : ogot=fzp_read(*ofp, obuf, FULL_CHUNK);
170 2 : ngot=fzp_read(*nfp, nbuf, FULL_CHUNK);
171 2 : if(ogot!=ngot) return 0;
172 20 : for(i=0; i<ogot; i++)
173 20 : if(obuf[i]!=nbuf[i]) return 0;
174 2 : if(ogot<FULL_CHUNK) break;
175 : }
176 :
177 : return 1;
178 : }
179 :
180 : #define PART_CHUNK 1024
181 :
182 4 : static int get_part_cksum(struct file *f, struct fzp **fzp)
183 : {
184 : MD5_CTX md5;
185 4 : int got=0;
186 : static char buf[PART_CHUNK];
187 : unsigned char checksum[MD5_DIGEST_LENGTH+1];
188 :
189 4 : if(*fzp) fzp_seek(*fzp, 0, SEEK_SET);
190 4 : else if(!(*fzp=fzp_open(f->path, "rb")))
191 : {
192 0 : f->part_cksum=0;
193 0 : return 0;
194 : }
195 :
196 4 : if(!MD5_Init(&md5))
197 : {
198 0 : logp("MD5_Init() failed\n");
199 0 : return -1;
200 : }
201 :
202 4 : got=fzp_read(*fzp, buf, PART_CHUNK);
203 :
204 4 : if(!MD5_Update(&md5, buf, got))
205 : {
206 0 : logp("MD5_Update() failed\n");
207 0 : return -1;
208 : }
209 :
210 4 : if(!MD5_Final(checksum, &md5))
211 : {
212 0 : logp("MD5_Final() failed\n");
213 0 : return -1;
214 : }
215 :
216 4 : memcpy(&(f->part_cksum), checksum, sizeof(unsigned));
217 :
218 : // Try for a bit of efficiency - no need to calculate the full checksum
219 : // again if we already read the whole file.
220 4 : if(got<PART_CHUNK) f->full_cksum=f->part_cksum;
221 :
222 : return 0;
223 : }
224 :
225 0 : static int get_full_cksum(struct file *f, struct fzp **fzp)
226 : {
227 0 : size_t s=0;
228 : MD5_CTX md5;
229 : static char buf[FULL_CHUNK];
230 : unsigned char checksum[MD5_DIGEST_LENGTH+1];
231 :
232 0 : if(*fzp) fzp_seek(*fzp, 0, SEEK_SET);
233 0 : else if(!(*fzp=fzp_open(f->path, "rb")))
234 : {
235 0 : f->full_cksum=0;
236 0 : return 0;
237 : }
238 :
239 0 : if(!MD5_Init(&md5))
240 : {
241 0 : logp("MD5_Init() failed\n");
242 0 : return -1;
243 : }
244 :
245 0 : while((s=fzp_read(*fzp, buf, FULL_CHUNK))>0)
246 : {
247 0 : if(!MD5_Update(&md5, buf, s))
248 : {
249 0 : logp("MD5_Update() failed\n");
250 0 : return -1;
251 : }
252 0 : if(s<FULL_CHUNK) break;
253 : }
254 :
255 0 : if(!MD5_Final(checksum, &md5))
256 : {
257 0 : logp("MD5_Final() failed\n");
258 0 : return -1;
259 : }
260 :
261 0 : memcpy(&(f->full_cksum), checksum, sizeof(unsigned));
262 :
263 0 : return 0;
264 : }
265 :
266 : /* Make it atomic by linking to a temporary file, then moving it into place. */
267 1 : static int do_hardlink(struct file *o, struct file *n)
268 : {
269 1 : int ret=-1;
270 1 : char *tmppath=NULL;
271 1 : if(!(tmppath=prepend(o->path, ext)))
272 : {
273 0 : log_out_of_memory(__func__);
274 : goto end;
275 : }
276 1 : if(link(n->path, tmppath))
277 : {
278 0 : logp("Could not hardlink %s to %s: %s\n", tmppath, n->path,
279 0 : strerror(errno));
280 : goto end;
281 : }
282 1 : if((ret=do_rename(tmppath, o->path)))
283 : {
284 : // 'man 2 rename', says it should be safe to unlink tmppath:
285 : // "If newpath exists but the operation fails for some reason,
286 : // rename() guarantees to leave an instance of newpath in
287 : // place."
288 0 : if(unlink(tmppath))
289 0 : logp("Could not unlink %s\n", tmppath);
290 : goto end;
291 : }
292 : ret=0;
293 : end:
294 1 : free_w(&tmppath);
295 1 : return ret;
296 : }
297 :
298 : static void reset_old_file(struct file *oldfile, struct file *newfile,
299 : struct stat *info)
300 : {
301 : //printf("reset %s with %s %d\n", oldfile->path, newfile->path,
302 : // info->st_nlink);
303 0 : oldfile->nlink=info->st_nlink;
304 0 : free_w(&oldfile->path);
305 0 : oldfile->path=newfile->path;
306 0 : newfile->path=NULL;
307 : }
308 :
309 2 : static int check_files(struct mystruct *find, struct file *newfile,
310 0 : struct stat *info)
311 : {
312 2 : int found=0;
313 2 : struct fzp *nfp=NULL;
314 2 : struct fzp *ofp=NULL;
315 2 : struct file *f=NULL;
316 :
317 2 : for(f=find->files; f; f=f->next)
318 : {
319 : //printf(" against: '%s'\n", f->path);
320 2 : if(!f->path)
321 : {
322 : // If the full_match() function fails to open oldfile
323 : // (which could happen if burp deleted some old
324 : // directories), it will free path and set it to NULL.
325 : // Skip entries like this.
326 0 : continue;
327 : }
328 2 : if(newfile->dev!=f->dev)
329 : {
330 : // Different device.
331 0 : continue;
332 : }
333 2 : if(newfile->ino==f->ino)
334 : {
335 : // Same device, same inode, therefore these two files
336 : // are hardlinked to each other already.
337 : found++;
338 : break;
339 : }
340 2 : if((!newfile->part_cksum && get_part_cksum(newfile, &nfp))
341 2 : || (!f->part_cksum && get_part_cksum(f, &ofp)))
342 : {
343 : // Some error with md5sums Give up.
344 : return -1;
345 : }
346 2 : if(newfile->part_cksum!=f->part_cksum)
347 : {
348 0 : fzp_close(&ofp);
349 0 : continue;
350 : }
351 : //printf(" %s, %s\n", find->files->path, newfile->path);
352 : //printf(" part cksum matched\n");
353 :
354 2 : if((!newfile->full_cksum && get_full_cksum(newfile, &nfp))
355 2 : || (!f->full_cksum && get_full_cksum(f, &ofp)))
356 : {
357 : // Some error with md5sums Give up.
358 : return -1;
359 : }
360 2 : if(newfile->full_cksum!=f->full_cksum)
361 : {
362 0 : fzp_close(&ofp);
363 0 : continue;
364 : }
365 :
366 : //printf(" full cksum matched\n");
367 2 : if(!full_match(newfile, f, &nfp, &ofp))
368 : {
369 0 : fzp_close(&ofp);
370 0 : continue;
371 : }
372 : //printf(" full match\n");
373 : //printf("%s, %s\n", find->files->path, newfile->path);
374 :
375 : // If there are already enough links to this file, replace
376 : // our memory of it with the new file so that files later on
377 : // can link to the new one.
378 2 : if(f->nlink>=maxlinks)
379 : {
380 : // Just need to reset the path name and the number
381 : // of links, and pretend that it was found otherwise
382 : // NULL newfile will get added to the memory.
383 0 : reset_old_file(f, newfile, info);
384 0 : found++;
385 0 : break;
386 : }
387 :
388 2 : found++;
389 2 : count++;
390 :
391 2 : if(verbose) printf("%s\n", newfile->path);
392 :
393 : // Now hardlink it.
394 2 : if(makelinks)
395 : {
396 1 : if(do_hardlink(newfile, f))
397 : {
398 0 : count--;
399 0 : return -1;
400 : }
401 1 : f->nlink++;
402 : // Only count bytes as saved if we
403 : // removed the last link.
404 1 : if(newfile->nlink==1)
405 1 : savedbytes+=info->st_size;
406 : }
407 1 : else if(deletedups)
408 : {
409 0 : if(unlink(newfile->path))
410 : {
411 0 : logp("Could not delete %s: %s\n",
412 0 : newfile->path, strerror(errno));
413 : }
414 : else
415 : {
416 : // Only count bytes as saved if we removed the
417 : // last link.
418 0 : if(newfile->nlink==1)
419 0 : savedbytes+=info->st_size;
420 : }
421 : }
422 : else
423 : {
424 : // To be able to tell how many bytes
425 : // are saveable.
426 1 : savedbytes+=info->st_size;
427 : }
428 :
429 : break;
430 : }
431 2 : fzp_close(&nfp);
432 2 : fzp_close(&ofp);
433 :
434 2 : if(found)
435 : {
436 2 : free_w(&newfile->path);
437 2 : return 0;
438 : }
439 :
440 0 : if(add_file(find, newfile)) return -1;
441 :
442 0 : return 0;
443 : }
444 :
445 0 : static int looks_like_protocol1(const char *basedir)
446 : {
447 0 : int ret=-1;
448 0 : char *tmp=NULL;
449 0 : if(!(tmp=prepend_s(basedir, "current")))
450 : {
451 0 : log_out_of_memory(__func__);
452 0 : goto end;
453 : }
454 : // If there is a 'current' symlink here, we think it looks like a
455 : // protocol 1 backup.
456 0 : if(is_lnk_lstat(tmp)>0)
457 : {
458 : ret=1;
459 : goto end;
460 : }
461 0 : ret=0;
462 : end:
463 0 : free_w(&tmp);
464 0 : return ret;
465 : }
466 :
467 0 : static int get_link(const char *basedir, const char *lnk, char real[], size_t r)
468 : {
469 0 : readlink_w_in_dir(basedir, lnk, real, r);
470 : // Strip any trailing slash.
471 0 : if(real[strlen(real)-1]=='/')
472 0 : real[strlen(real)-1]='\0';
473 0 : return 0;
474 : }
475 :
476 0 : static int level_exclusion(int level, const char *fname,
477 : const char *working, const char *finishing)
478 : {
479 0 : if(level==0)
480 : {
481 : /* Be careful not to try to dedup the lockfiles.
482 : The lock actually gets lost if you open one to do a
483 : checksum
484 : and then close it. This caused me major headaches to
485 : figure out. */
486 0 : if(!strcmp(fname, LOCKFILE_NAME)
487 0 : || !strcmp(fname, BEDUP_LOCKFILE_NAME))
488 : return 1;
489 :
490 : /* Skip places where backups are going on. */
491 0 : if(!strcmp(fname, working)
492 0 : || !strcmp(fname, finishing))
493 : return 1;
494 :
495 0 : if(!strcmp(fname, "deleteme"))
496 : return 1;
497 : }
498 0 : else if(level==1)
499 : {
500 : // Do not dedup stuff that might be appended to later.
501 0 : if(!strncmp(fname, "log", strlen("log"))
502 0 : || !strncmp(fname, "verifylog", strlen("verifylog"))
503 0 : || !strncmp(fname, "restorelog", strlen("restorelog")))
504 : return 1;
505 : }
506 0 : return 0;
507 : }
508 :
509 : // Return 0 for directory processed, -1 for error, 1 for not processed.
510 2 : static int process_dir(const char *oldpath, const char *newpath,
511 : int burp_mode, int level)
512 : {
513 2 : int ret=-1;
514 2 : DIR *dirp=NULL;
515 2 : char *path=NULL;
516 : struct stat info;
517 2 : struct dirent *dirinfo=NULL;
518 : struct file newfile;
519 2 : struct mystruct *find=NULL;
520 : static char working[256]="";
521 : static char finishing[256]="";
522 :
523 2 : newfile.path=NULL;
524 :
525 2 : if(!(path=prepend_s(oldpath, newpath))) goto end;
526 :
527 2 : if(burp_mode && level==0)
528 : {
529 0 : if(get_link(path, "working", working, sizeof(working))
530 0 : || get_link(path, "finishing", finishing, sizeof(finishing)))
531 : goto end;
532 0 : if(!looks_like_protocol1(path))
533 : {
534 0 : logp("%s does not look like a protocol 1 storage directory - skipping\n", path);
535 0 : ret=1;
536 0 : goto end;
537 : }
538 : }
539 :
540 2 : if(!(dirp=opendir(path)))
541 : {
542 0 : logp("Could not opendir '%s': %s\n", path, strerror(errno));
543 0 : ret=1;
544 0 : goto end;
545 : }
546 10 : while((dirinfo=readdir(dirp)))
547 : {
548 8 : if(!strcmp(dirinfo->d_name, ".")
549 6 : || !strcmp(dirinfo->d_name, ".."))
550 4 : continue;
551 :
552 : //printf("try %s\n", dirinfo->d_name);
553 :
554 4 : if(burp_mode
555 0 : && level_exclusion(level, dirinfo->d_name,
556 : working, finishing))
557 0 : continue;
558 :
559 4 : free_w(&newfile.path);
560 4 : if(!(newfile.path=prepend_s(path, dirinfo->d_name)))
561 : goto end;
562 :
563 8 : if(lstat(newfile.path, &info))
564 0 : continue;
565 :
566 4 : if(S_ISDIR(info.st_mode))
567 : {
568 0 : if(process_dir(path, dirinfo->d_name,
569 : burp_mode, level+1))
570 : goto end;
571 0 : continue;
572 : }
573 4 : else if(!S_ISREG(info.st_mode)
574 4 : || !info.st_size) // ignore zero-length files
575 0 : continue;
576 :
577 4 : newfile.dev=info.st_dev;
578 4 : newfile.ino=info.st_ino;
579 4 : newfile.nlink=info.st_nlink;
580 4 : newfile.full_cksum=0;
581 4 : newfile.part_cksum=0;
582 4 : newfile.next=NULL;
583 :
584 4 : if((find=find_key(info.st_size)))
585 : {
586 : //printf("check %d: %s\n", info.st_size, newfile.path);
587 2 : if(check_files(find, &newfile, &info))
588 : goto end;
589 : }
590 : else
591 : {
592 : //printf("add: %s\n", newfile.path);
593 2 : if(add_key(info.st_size, &newfile))
594 : goto end;
595 : }
596 : }
597 : ret=0;
598 : end:
599 2 : if(dirp) closedir(dirp);
600 2 : free_w(&newfile.path);
601 2 : free_w(&path);
602 2 : return ret;
603 : }
604 :
605 0 : static void sighandler(__attribute__ ((unused)) int signum)
606 : {
607 0 : locks_release_and_free(&locklist);
608 0 : exit(1);
609 : }
610 :
611 0 : static int is_regular_file(const char *clientconfdir, const char *file)
612 : {
613 : struct stat statp;
614 0 : char *fullpath=NULL;
615 0 : if(!(fullpath=prepend_s(clientconfdir, file)))
616 : return 0;
617 0 : if(lstat(fullpath, &statp))
618 : {
619 0 : free_w(&fullpath);
620 0 : return 0;
621 : }
622 0 : free_w(&fullpath);
623 0 : return S_ISREG(statp.st_mode);
624 : }
625 :
626 0 : static int in_group(struct strlist *grouplist, const char *dedup_group)
627 : {
628 : struct strlist *g;
629 :
630 0 : for(g=grouplist; g; g=g->next)
631 0 : if(!strcmp(g->path, dedup_group)) return 1;
632 :
633 : return 0;
634 : }
635 :
636 0 : static int iterate_over_clients(struct conf **globalcs,
637 : struct strlist *grouplist)
638 : {
639 0 : int ret=0;
640 0 : DIR *dirp=NULL;
641 0 : struct conf **cconfs=NULL;
642 0 : struct dirent *dirinfo=NULL;
643 0 : const char *globalclientconfdir=get_string(globalcs[OPT_CLIENTCONFDIR]);
644 :
645 0 : if(!(cconfs=confs_alloc())) return -1;
646 0 : if(confs_init(cconfs)) return -1;
647 :
648 0 : if(!(dirp=opendir(globalclientconfdir)))
649 : {
650 0 : logp("Could not opendir '%s': %s\n",
651 0 : globalclientconfdir, strerror(errno));
652 0 : return 0;
653 : }
654 0 : while((dirinfo=readdir(dirp)))
655 : {
656 0 : char *lockfile=NULL;
657 0 : char *lockfilebase=NULL;
658 0 : char *client_lockdir=NULL;
659 0 : struct lock *lock=NULL;
660 :
661 0 : if(dirinfo->d_ino==0
662 : // looks_like...() also avoids '.' and '..'.
663 0 : || looks_like_tmp_or_hidden_file(dirinfo->d_name)
664 0 : || !is_regular_file(globalclientconfdir, dirinfo->d_name))
665 0 : continue;
666 :
667 0 : confs_free_content(cconfs);
668 0 : if(confs_init(cconfs)) return -1;
669 :
670 0 : if(set_string(cconfs[OPT_CNAME], dirinfo->d_name))
671 : return -1;
672 :
673 0 : if(conf_load_clientconfdir(globalcs, cconfs))
674 : {
675 0 : logp("could not load config for client %s\n",
676 : dirinfo->d_name);
677 0 : return 0;
678 : }
679 :
680 0 : if(grouplist)
681 : {
682 0 : const char *dedup_group=
683 0 : get_string(cconfs[OPT_DEDUP_GROUP]);
684 0 : if(!dedup_group
685 0 : || !in_group(grouplist, dedup_group))
686 0 : continue;
687 : }
688 :
689 0 : if(!(client_lockdir=get_string(cconfs[OPT_CLIENT_LOCKDIR])))
690 0 : client_lockdir=get_string(cconfs[OPT_DIRECTORY]);
691 :
692 0 : if(!(lockfilebase=prepend_s(client_lockdir, dirinfo->d_name))
693 0 : || !(lockfile=prepend_s(lockfilebase, BEDUP_LOCKFILE_NAME)))
694 : {
695 0 : free_w(&lockfilebase);
696 0 : free_w(&lockfile);
697 0 : ret=-1;
698 0 : break;
699 : }
700 0 : free_w(&lockfilebase);
701 :
702 0 : if(!(lock=lock_alloc_and_init(lockfile)))
703 : {
704 : ret=-1;
705 : break;
706 : }
707 0 : lock_get(lock);
708 0 : free_w(&lockfile);
709 :
710 0 : if(lock->status!=GET_LOCK_GOT)
711 : {
712 0 : logp("Could not get %s\n", lock->path);
713 0 : continue;
714 : }
715 0 : logp("Got %s\n", lock->path);
716 :
717 : // Remember that we got that lock.
718 0 : lock_add_to_list(&locklist, lock);
719 :
720 0 : switch(process_dir(get_string(cconfs[OPT_DIRECTORY]),
721 : dirinfo->d_name,
722 : 1 /* burp mode */, 0 /* level */))
723 : {
724 0 : case 0: ccount++;
725 0 : case 1: continue;
726 : default: ret=-1; break;
727 : }
728 : break;
729 : }
730 0 : closedir(dirp);
731 :
732 0 : locks_release_and_free(&locklist);
733 :
734 0 : confs_free(&cconfs);
735 :
736 0 : return ret;
737 : }
738 :
739 0 : static int process_from_conf(const char *configfile, char **groups)
740 : {
741 0 : int ret=-1;
742 0 : struct conf **globalcs=NULL;
743 0 : struct strlist *grouplist=NULL;
744 0 : struct lock *globallock=NULL;
745 :
746 0 : signal(SIGABRT, &sighandler);
747 0 : signal(SIGTERM, &sighandler);
748 0 : signal(SIGINT, &sighandler);
749 :
750 0 : if(*groups)
751 : {
752 0 : char *tok=NULL;
753 0 : if((tok=strtok(*groups, ",\n")))
754 : {
755 : do
756 : {
757 0 : if(strlist_add(&grouplist, tok, 1))
758 : {
759 0 : log_out_of_memory(__func__);
760 0 : goto end;
761 : }
762 0 : } while((tok=strtok(NULL, ",\n")));
763 : }
764 0 : if(!grouplist)
765 : {
766 0 : logp("unable to read list of groups\n");
767 0 : goto end;
768 : }
769 : }
770 :
771 : // Read directories from config files, and get locks.
772 0 : if(!(globalcs=confs_alloc())
773 0 : || confs_init(globalcs)
774 0 : || conf_load_global_only(configfile, globalcs))
775 : goto end;
776 :
777 0 : if(get_e_burp_mode(globalcs[OPT_BURP_MODE])!=BURP_MODE_SERVER)
778 : {
779 0 : logp("%s is not a server config file\n", configfile);
780 0 : goto end;
781 : }
782 0 : logp("Dedup clients from %s\n",
783 0 : get_string(globalcs[OPT_CLIENTCONFDIR]));
784 0 : maxlinks=get_int(globalcs[OPT_MAX_HARDLINKS]);
785 0 : if(grouplist)
786 : {
787 0 : struct strlist *g=NULL;
788 0 : logp("in dedup groups:\n");
789 0 : for(g=grouplist; g; g=g->next)
790 0 : logp("%s\n", g->path);
791 : }
792 : else
793 : {
794 0 : char *lockpath=NULL;
795 0 : const char *opt_lockfile=confs_get_lockfile(globalcs);
796 : // Only get the global lock when doing a global run.
797 : // If you are doing individual groups, you are likely
798 : // to want to do many different dedup jobs and a
799 : // global lock would get in the way.
800 0 : if(!(lockpath=prepend(opt_lockfile, ".bedup"))
801 0 : || !(globallock=lock_alloc_and_init(lockpath)))
802 : goto end;
803 0 : lock_get(globallock);
804 0 : if(globallock->status!=GET_LOCK_GOT)
805 : {
806 0 : logp("Could not get lock %s (%d)\n", lockpath,
807 : globallock->status);
808 0 : free_w(&lockpath);
809 0 : goto end;
810 : }
811 0 : logp("Got %s\n", lockpath);
812 : }
813 0 : ret=iterate_over_clients(globalcs, grouplist);
814 : end:
815 0 : confs_free(&globalcs);
816 0 : lock_release(globallock);
817 0 : lock_free(&globallock);
818 0 : strlists_free(&grouplist);
819 0 : return ret;
820 : }
821 :
822 2 : static int process_from_command_line(int argc, char *argv[])
823 : {
824 : int i;
825 4 : for(i=optind; i<argc; i++)
826 : {
827 : // Strip trailing slashes, for tidiness.
828 2 : if(argv[i][strlen(argv[i])-1]=='/')
829 0 : argv[i][strlen(argv[i])-1]='\0';
830 2 : if(process_dir("", argv[i],
831 : 0 /* not burp mode */, 0 /* level */))
832 : return 1;
833 : }
834 : return 0;
835 : }
836 :
837 2 : static int usage(void)
838 : {
839 2 : logfmt("\nUsage: %s [options]\n", prog);
840 2 : logfmt("\n");
841 2 : logfmt(" Options:\n");
842 2 : logfmt(" -c <path> Path to config file (default: %s).\n", config_default_path());
843 2 : logfmt(" -g <list of group names> Only run on the directories of clients that\n");
844 2 : logfmt(" are in one of the groups specified.\n");
845 2 : logfmt(" The list is comma-separated. To put a client in a\n");
846 2 : logfmt(" group, use the 'dedup_group' option in the client\n");
847 2 : logfmt(" configuration file on the server.\n");
848 2 : logfmt(" -h|-? Print this text and exit.\n");
849 2 : logfmt(" -d Delete any duplicate files found.\n");
850 2 : logfmt(" (non-%s mode only)\n", PACKAGE_TARNAME);
851 2 : logfmt(" -l Hard link any duplicate files found.\n");
852 2 : logfmt(" -m <number> Maximum number of hard links to a single file.\n");
853 2 : logfmt(" (non-%s mode only - in burp mode, use the\n", PACKAGE_TARNAME);
854 2 : logfmt(" max_hardlinks option in the configuration file)\n");
855 2 : logfmt(" The default is %d. On ext3, the maximum number\n", DEF_MAX_LINKS);
856 2 : logfmt(" of links possible is 32000, but space is needed\n");
857 2 : logfmt(" for the normal operation of %s.\n", PACKAGE_TARNAME);
858 2 : logfmt(" -n <list of directories> Non-%s mode. Deduplicate any (set of) directories.\n", PACKAGE_TARNAME);
859 2 : logfmt(" -v Print duplicate paths.\n");
860 2 : logfmt(" -V Print version and exit.\n");
861 2 : logfmt("\n");
862 2 : logfmt("By default, %s will read %s and deduplicate client storage\n", prog, config_default_path());
863 2 : logfmt("directories using special knowledge of the structure.\n");
864 2 : logfmt("\n");
865 2 : logfmt("With '-n', this knowledge is turned off and you have to specify the directories\n");
866 2 : logfmt("to deduplicate on the command line. Running with '-n' is therefore dangerous\n");
867 2 : logfmt("if you are deduplicating %s storage directories.\n\n", PACKAGE_TARNAME);
868 2 : return 1;
869 : }
870 :
871 13 : int run_bedup(int argc, char *argv[])
872 : {
873 13 : int ret=0;
874 13 : int option=0;
875 13 : int nonburp=0;
876 13 : char *groups=NULL;
877 13 : int givenconfigfile=0;
878 13 : const char *configfile=NULL;
879 :
880 13 : configfile=config_default_path();
881 13 : snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
882 :
883 28 : while((option=getopt(argc, argv, "c:dg:hlm:nvV?"))!=-1)
884 : {
885 18 : switch(option)
886 : {
887 : case 'c':
888 1 : configfile=optarg;
889 1 : givenconfigfile=1;
890 1 : break;
891 : case 'd':
892 2 : deletedups=1;
893 2 : break;
894 : case 'g':
895 1 : groups=optarg;
896 1 : break;
897 : case 'l':
898 2 : makelinks=1;
899 2 : break;
900 : case 'm':
901 4 : maxlinks=atoi(optarg);
902 2 : break;
903 : case 'n':
904 6 : nonburp=1;
905 6 : break;
906 : case 'V':
907 1 : logfmt("%s-%s\n", prog, PACKAGE_VERSION);
908 1 : return 0;
909 : case 'v':
910 1 : verbose=1;
911 1 : break;
912 : case 'h':
913 : case '?':
914 2 : return usage();
915 : }
916 : }
917 :
918 10 : if(nonburp && givenconfigfile)
919 : {
920 1 : logp("-n and -c options are mutually exclusive\n");
921 1 : return 1;
922 : }
923 9 : if(nonburp && groups)
924 : {
925 1 : logp("-n and -g options are mutually exclusive\n");
926 1 : return 1;
927 : }
928 8 : if(!nonburp && maxlinks!=DEF_MAX_LINKS)
929 : {
930 1 : logp("-m option is specified via the configuration file in %s mode (max_hardlinks=)\n", PACKAGE_TARNAME);
931 1 : return 1;
932 : }
933 7 : if(deletedups && makelinks)
934 : {
935 1 : logp("-d and -l options are mutually exclusive\n");
936 1 : return 1;
937 : }
938 6 : if(deletedups && !nonburp)
939 : {
940 1 : logp("-d option requires -n option\n");
941 1 : return 1;
942 : }
943 :
944 5 : if(optind>=argc)
945 : {
946 1 : if(nonburp)
947 : {
948 1 : logp("No directories found after options\n");
949 1 : return 1;
950 : }
951 : }
952 : else
953 : {
954 4 : if(!nonburp)
955 : {
956 1 : logp("Do not specify extra arguments.\n");
957 1 : return 1;
958 : }
959 : }
960 :
961 3 : if(maxlinks<2)
962 : {
963 1 : logp("The argument to -m needs to be greater than 1.\n");
964 1 : return 1;
965 : }
966 :
967 2 : if(nonburp)
968 : {
969 : // Read directories from command line.
970 2 : if(process_from_command_line(argc, argv))
971 0 : ret=1;
972 : }
973 : else
974 : {
975 0 : if(process_from_conf(configfile, &groups))
976 0 : ret=1;
977 : }
978 :
979 2 : if(!nonburp)
980 : {
981 0 : logp("%d client storages scanned\n", ccount);
982 : }
983 2 : logp("%" PRIu64 " duplicate %s found\n",
984 2 : count, count==1?"file":"files");
985 6 : logp("%" PRIu64 " bytes %s%s\n",
986 3 : savedbytes, (makelinks || deletedups)?"saved":"saveable",
987 : bytes_to_human(savedbytes));
988 2 : mystruct_delete_all();
989 2 : return ret;
990 : }
|
