Line data Source code
1 : #include "../../burp.h"
2 : #include "../../alloc.h"
3 : #include "../../conf.h"
4 : #include "../../conffile.h"
5 : #include "../../handy.h"
6 : #include "../../fsops.h"
7 : #include "../../fzp.h"
8 : #include "../../lock.h"
9 : #include "../../log.h"
10 : #include "../../prepend.h"
11 : #include "../../strlist.h"
12 : #include "bedup.h"
13 :
14 : #include <uthash.h>
15 :
16 : #define LOCKFILE_NAME "lockfile"
17 : #define BEDUP_LOCKFILE_NAME "lockfile.bedup"
18 :
19 : #define DEF_MAX_LINKS 10000
20 :
21 : static int makelinks=0;
22 : static int deletedups=0;
23 :
24 : static uint64_t savedbytes=0;
25 : static uint64_t count=0;
26 : static int ccount=0;
27 :
28 : static struct lock *locklist=NULL;
29 :
30 : static int verbose=0;
31 :
32 : static unsigned int maxlinks=DEF_MAX_LINKS;
33 : static char ext[16]="";
34 :
35 : typedef struct file file_t;
36 :
37 : struct file
38 : {
39 : char *path;
40 : dev_t dev;
41 : ino_t ino;
42 : nlink_t nlink;
43 : uint64_t full_cksum;
44 : uint64_t part_cksum;
45 : file_t *next;
46 : };
47 :
48 : struct mystruct
49 : {
50 : off_t st_size;
51 : file_t *files;
52 : UT_hash_handle hh;
53 : };
54 :
55 : struct mystruct *myfiles=NULL;
56 :
57 9 : static struct mystruct *find_key(off_t st_size)
58 : {
59 : struct mystruct *s;
60 :
61 9 : HASH_FIND_INT(myfiles, &st_size, s);
62 9 : return s;
63 : }
64 :
65 3 : static int add_file(struct mystruct *s, struct file *f)
66 : {
67 : struct file *newfile;
68 3 : if(!(newfile=(struct file *)malloc_w(sizeof(struct file), __func__)))
69 : return -1;
70 3 : memcpy(newfile, f, sizeof(struct file));
71 3 : f->path=NULL;
72 3 : newfile->next=s->files;
73 3 : s->files=newfile;
74 : return 0;
75 : }
76 :
77 3 : static int add_key(off_t st_size, struct file *f)
78 : {
79 : struct mystruct *s;
80 :
81 3 : if(!(s=(struct mystruct *)malloc_w(sizeof(struct mystruct), __func__)))
82 : return -1;
83 3 : s->st_size=st_size;
84 3 : s->files=NULL;
85 3 : if(add_file(s, f)) return -1;
86 : //printf("HASH ADD %d\n", st_size);
87 3 : HASH_ADD_INT(myfiles, st_size, s);
88 : return 0;
89 : }
90 :
91 : static void file_free_content(struct file *file)
92 : {
93 : if(!file) return;
94 3 : free_w(&file->path);
95 : }
96 :
97 3 : static void file_free(struct file **file)
98 : {
99 3 : if(!file || !*file) return;
100 6 : file_free_content(*file);
101 3 : free_v((void **)file);
102 : }
103 :
104 3 : static void files_free(struct file **files)
105 : {
106 : struct file *f;
107 : struct file *fhead;
108 3 : if(!files || !*files) return;
109 : fhead=*files;
110 6 : while(fhead)
111 : {
112 3 : f=fhead;
113 3 : fhead=fhead->next;
114 3 : file_free(&f);
115 : }
116 : }
117 :
118 : static void mystruct_free_content(struct mystruct *mystruct)
119 : {
120 : if(!mystruct) return;
121 3 : files_free(&mystruct->files);
122 : }
123 :
124 3 : static void mystruct_free(struct mystruct **mystruct)
125 : {
126 3 : if(!mystruct || !*mystruct) return;
127 6 : mystruct_free_content(*mystruct);
128 3 : free_v((void **)mystruct);
129 : }
130 :
131 3 : static void mystruct_delete_all(void)
132 : {
133 : struct mystruct *tmp;
134 : struct mystruct *mystruct;
135 :
136 6 : HASH_ITER(hh, myfiles, mystruct, tmp)
137 : {
138 3 : HASH_DEL(myfiles, mystruct);
139 3 : mystruct_free(&mystruct);
140 : }
141 3 : myfiles=NULL;
142 3 : }
143 :
144 : #define FULL_CHUNK 4096
145 :
146 6 : static int full_match(struct file *o, struct file *n,
147 : struct fzp **ofp, struct fzp **nfp)
148 : {
149 : size_t ogot;
150 : size_t ngot;
151 6 : unsigned int i=0;
152 : static char obuf[FULL_CHUNK];
153 : static char nbuf[FULL_CHUNK];
154 :
155 6 : if(*ofp) fzp_seek(*ofp, 0, SEEK_SET);
156 0 : else if(!(*ofp=fzp_open(o->path, "rb")))
157 : {
158 : // Blank this entry so that it can be ignored from
159 : // now on.
160 0 : free_w(&o->path);
161 : return 0;
162 : }
163 :
164 6 : if(*nfp) fzp_seek(*nfp, 0, SEEK_SET);
165 3 : else if(!(*nfp=fzp_open(n->path, "rb"))) return 0;
166 :
167 : while(1)
168 : {
169 6 : ogot=fzp_read(*ofp, obuf, FULL_CHUNK);
170 6 : ngot=fzp_read(*nfp, nbuf, FULL_CHUNK);
171 6 : if(ogot!=ngot) return 0;
172 60 : for(i=0; i<ogot; i++)
173 60 : if(obuf[i]!=nbuf[i]) return 0;
174 6 : if(ogot<FULL_CHUNK) break;
175 : }
176 :
177 : return 1;
178 : }
179 :
180 : #define PART_CHUNK 1024
181 :
182 9 : static int get_part_cksum(struct file *f, struct fzp **fzp)
183 : {
184 : MD5_CTX md5;
185 9 : int got=0;
186 : static char buf[PART_CHUNK];
187 : unsigned char checksum[MD5_DIGEST_LENGTH+1];
188 :
189 9 : if(*fzp) fzp_seek(*fzp, 0, SEEK_SET);
190 9 : else if(!(*fzp=fzp_open(f->path, "rb")))
191 : {
192 0 : f->part_cksum=0;
193 0 : return 0;
194 : }
195 :
196 9 : if(!MD5_Init(&md5))
197 : {
198 0 : logp("MD5_Init() failed\n");
199 0 : return -1;
200 : }
201 :
202 9 : got=fzp_read(*fzp, buf, PART_CHUNK);
203 :
204 9 : if(!MD5_Update(&md5, buf, got))
205 : {
206 0 : logp("MD5_Update() failed\n");
207 0 : return -1;
208 : }
209 :
210 9 : if(!MD5_Final(checksum, &md5))
211 : {
212 0 : logp("MD5_Final() failed\n");
213 0 : return -1;
214 : }
215 :
216 9 : memcpy(&(f->part_cksum), checksum, sizeof(unsigned));
217 :
218 : // Try for a bit of efficiency - no need to calculate the full checksum
219 : // again if we already read the whole file.
220 9 : if(got<PART_CHUNK) f->full_cksum=f->part_cksum;
221 :
222 : return 0;
223 : }
224 :
225 0 : static int get_full_cksum(struct file *f, struct fzp **fzp)
226 : {
227 0 : size_t s=0;
228 : MD5_CTX md5;
229 : static char buf[FULL_CHUNK];
230 : unsigned char checksum[MD5_DIGEST_LENGTH+1];
231 :
232 0 : if(*fzp) fzp_seek(*fzp, 0, SEEK_SET);
233 0 : else if(!(*fzp=fzp_open(f->path, "rb")))
234 : {
235 0 : f->full_cksum=0;
236 0 : return 0;
237 : }
238 :
239 0 : if(!MD5_Init(&md5))
240 : {
241 0 : logp("MD5_Init() failed\n");
242 0 : return -1;
243 : }
244 :
245 0 : while((s=fzp_read(*fzp, buf, FULL_CHUNK))>0)
246 : {
247 0 : if(!MD5_Update(&md5, buf, s))
248 : {
249 0 : logp("MD5_Update() failed\n");
250 0 : return -1;
251 : }
252 0 : if(s<FULL_CHUNK) break;
253 : }
254 :
255 0 : if(!MD5_Final(checksum, &md5))
256 : {
257 0 : logp("MD5_Final() failed\n");
258 0 : return -1;
259 : }
260 :
261 0 : memcpy(&(f->full_cksum), checksum, sizeof(unsigned));
262 :
263 0 : return 0;
264 : }
265 :
266 : /* Make it atomic by linking to a temporary file, then moving it into place. */
267 3 : static int do_hardlink(struct file *o, struct file *n)
268 : {
269 3 : int ret=-1;
270 3 : char *tmppath=NULL;
271 3 : if(!(tmppath=prepend(o->path, ext)))
272 : {
273 0 : log_out_of_memory(__func__);
274 : goto end;
275 : }
276 3 : if(link(n->path, tmppath))
277 : {
278 0 : logp("Could not hardlink %s to %s: %s\n", tmppath, n->path,
279 0 : strerror(errno));
280 : goto end;
281 : }
282 3 : if((ret=do_rename(tmppath, o->path)))
283 : {
284 : // 'man 2 rename', says it should be safe to unlink tmppath:
285 : // "If newpath exists but the operation fails for some reason,
286 : // rename() guarantees to leave an instance of newpath in
287 : // place."
288 0 : if(unlink(tmppath))
289 0 : logp("Could not unlink %s\n", tmppath);
290 : goto end;
291 : }
292 : ret=0;
293 : end:
294 3 : free_w(&tmppath);
295 3 : return ret;
296 : }
297 :
298 2 : static void reset_old_file(struct file *oldfile, struct file *newfile,
299 : struct stat *info)
300 : {
301 : //printf("reset %s with %s %d\n", oldfile->path, newfile->path,
302 : // info->st_nlink);
303 : struct file *next;
304 :
305 2 : next=oldfile->next;
306 2 : free_w(&oldfile->path);
307 2 : memcpy(oldfile, newfile, sizeof(struct file));
308 2 : oldfile->next=next;
309 2 : newfile->path=NULL;
310 2 : }
311 :
312 6 : static int check_files(struct mystruct *find, struct file *newfile,
313 : struct stat *info)
314 : {
315 6 : int found=0;
316 6 : struct fzp *nfp=NULL;
317 6 : struct fzp *ofp=NULL;
318 6 : struct file *f=NULL;
319 :
320 6 : for(f=find->files; f; f=f->next)
321 : {
322 : //printf(" against: '%s'\n", f->path);
323 6 : if(!f->path)
324 : {
325 : // If the full_match() function fails to open oldfile
326 : // (which could happen if burp deleted some old
327 : // directories), it will free path and set it to NULL.
328 : // Skip entries like this.
329 0 : continue;
330 : }
331 6 : if(newfile->dev!=f->dev)
332 : {
333 : // Different device.
334 0 : continue;
335 : }
336 6 : if(newfile->ino==f->ino)
337 : {
338 : // Same device, same inode, therefore these two files
339 : // are hardlinked to each other already.
340 : found++;
341 : break;
342 : }
343 6 : if(newfile->nlink>=maxlinks) {
344 : // This new file file has enough links. Just leave it
345 : // as it is to avoid undoing all these hardlinks.
346 : found++;
347 : break;
348 : }
349 6 : if((!newfile->part_cksum && get_part_cksum(newfile, &nfp))
350 6 : || (!f->part_cksum && get_part_cksum(f, &ofp)))
351 : {
352 : // Some error with md5sums Give up.
353 : return -1;
354 : }
355 6 : if(newfile->part_cksum!=f->part_cksum)
356 : {
357 0 : fzp_close(&ofp);
358 0 : continue;
359 : }
360 : //printf(" %s, %s\n", find->files->path, newfile->path);
361 : //printf(" part cksum matched\n");
362 :
363 6 : if((!newfile->full_cksum && get_full_cksum(newfile, &nfp))
364 6 : || (!f->full_cksum && get_full_cksum(f, &ofp)))
365 : {
366 : // Some error with md5sums Give up.
367 : return -1;
368 : }
369 6 : if(newfile->full_cksum!=f->full_cksum)
370 : {
371 0 : fzp_close(&ofp);
372 0 : continue;
373 : }
374 :
375 : //printf(" full cksum matched\n");
376 6 : if(!full_match(newfile, f, &nfp, &ofp))
377 : {
378 0 : fzp_close(&ofp);
379 0 : continue;
380 : }
381 : //printf(" full match\n");
382 : //printf("%s, %s\n", find->files->path, newfile->path);
383 :
384 : // If there are already enough links to this file, replace
385 : // our memory of it with the new file so that files later on
386 : // can link to the new one.
387 6 : if(f->nlink>=maxlinks)
388 : {
389 : // Just need to reset the path name and the number
390 : // of links, and pretend that it was found otherwise
391 : // NULL newfile will get added to the memory.
392 2 : reset_old_file(f, newfile, info);
393 2 : found++;
394 : break;
395 : }
396 :
397 4 : found++;
398 4 : count++;
399 :
400 4 : if(verbose) printf("%s\n", newfile->path);
401 :
402 : // Now hardlink it.
403 4 : if(makelinks)
404 : {
405 3 : if(do_hardlink(newfile, f))
406 : {
407 0 : count--;
408 : return -1;
409 : }
410 3 : f->nlink++;
411 : // Only count bytes as saved if we
412 : // removed the last link.
413 3 : if(newfile->nlink==1)
414 3 : savedbytes+=info->st_size;
415 : }
416 1 : else if(deletedups)
417 : {
418 0 : if(unlink(newfile->path))
419 : {
420 0 : logp("Could not delete %s: %s\n",
421 0 : newfile->path, strerror(errno));
422 : }
423 : else
424 : {
425 : // Only count bytes as saved if we removed the
426 : // last link.
427 0 : if(newfile->nlink==1)
428 0 : savedbytes+=info->st_size;
429 : }
430 : }
431 : else
432 : {
433 : // To be able to tell how many bytes
434 : // are saveable.
435 1 : savedbytes+=info->st_size;
436 : }
437 :
438 : break;
439 : }
440 6 : fzp_close(&nfp);
441 6 : fzp_close(&ofp);
442 :
443 6 : if(found)
444 : {
445 6 : free_w(&newfile->path);
446 : return 0;
447 : }
448 :
449 0 : if(add_file(find, newfile)) return -1;
450 :
451 : return 0;
452 : }
453 :
454 0 : static int looks_like_protocol1(const char *basedir)
455 : {
456 0 : int ret=-1;
457 0 : char *tmp=NULL;
458 0 : if(!(tmp=prepend_s(basedir, "current")))
459 : {
460 0 : log_out_of_memory(__func__);
461 0 : goto end;
462 : }
463 : // If there is a 'current' symlink here, we think it looks like a
464 : // protocol 1 backup.
465 0 : if(is_lnk_lstat(tmp)>0)
466 : {
467 : ret=1;
468 : goto end;
469 : }
470 0 : ret=0;
471 : end:
472 0 : free_w(&tmp);
473 0 : return ret;
474 : }
475 :
476 0 : static int get_link(const char *basedir, const char *lnk, char real[], size_t r)
477 : {
478 0 : readlink_w_in_dir(basedir, lnk, real, r);
479 : // Strip any trailing slash.
480 0 : if(real[strlen(real)-1]=='/')
481 0 : real[strlen(real)-1]='\0';
482 0 : return 0;
483 : }
484 :
485 0 : static int level_exclusion(int level, const char *fname,
486 : const char *working, const char *finishing)
487 : {
488 0 : if(level==0)
489 : {
490 : /* Be careful not to try to dedup the lockfiles.
491 : The lock actually gets lost if you open one to do a
492 : checksum
493 : and then close it. This caused me major headaches to
494 : figure out. */
495 0 : if(!strcmp(fname, LOCKFILE_NAME)
496 0 : || !strcmp(fname, BEDUP_LOCKFILE_NAME))
497 : return 1;
498 :
499 : /* Skip places where backups are going on. */
500 0 : if(!strcmp(fname, working)
501 0 : || !strcmp(fname, finishing))
502 : return 1;
503 :
504 0 : if(!strcmp(fname, "deleteme"))
505 : return 1;
506 : }
507 0 : else if(level==1)
508 : {
509 : // Do not dedup stuff that might be appended to later.
510 0 : if(!strncmp(fname, "log", strlen("log"))
511 0 : || !strncmp(fname, "verifylog", strlen("verifylog"))
512 0 : || !strncmp(fname, "restorelog", strlen("restorelog")))
513 : return 1;
514 : }
515 0 : return 0;
516 : }
517 :
518 : // Return 0 for directory processed, -1 for error, 1 for not processed.
519 3 : static int process_dir(const char *oldpath, const char *newpath,
520 : int burp_mode, int level)
521 : {
522 3 : int ret=-1;
523 3 : DIR *dirp=NULL;
524 3 : char *path=NULL;
525 : struct stat info;
526 3 : struct dirent *dirinfo=NULL;
527 : struct file newfile;
528 3 : struct mystruct *find=NULL;
529 : static char working[256]="";
530 : static char finishing[256]="";
531 :
532 3 : newfile.path=NULL;
533 :
534 3 : if(!(path=prepend_s(oldpath, newpath))) goto end;
535 :
536 3 : if(burp_mode && level==0)
537 : {
538 0 : if(get_link(path, "working", working, sizeof(working))
539 0 : || get_link(path, "finishing", finishing, sizeof(finishing)))
540 : goto end;
541 0 : if(!looks_like_protocol1(path))
542 : {
543 0 : logp("%s does not look like a protocol 1 storage directory - skipping\n", path);
544 0 : ret=1;
545 0 : goto end;
546 : }
547 : }
548 :
549 3 : if(!(dirp=opendir(path)))
550 : {
551 0 : logp("Could not opendir '%s': %s\n", path, strerror(errno));
552 0 : ret=1;
553 0 : goto end;
554 : }
555 18 : while((dirinfo=readdir(dirp)))
556 : {
557 15 : if(!strcmp(dirinfo->d_name, ".")
558 12 : || !strcmp(dirinfo->d_name, ".."))
559 6 : continue;
560 :
561 : //printf("try %s\n", dirinfo->d_name);
562 :
563 9 : if(burp_mode
564 0 : && level_exclusion(level, dirinfo->d_name,
565 : working, finishing))
566 0 : continue;
567 :
568 9 : free_w(&newfile.path);
569 9 : if(!(newfile.path=prepend_s(path, dirinfo->d_name)))
570 : goto end;
571 :
572 18 : if(lstat(newfile.path, &info))
573 0 : continue;
574 :
575 9 : if(S_ISDIR(info.st_mode))
576 : {
577 0 : if(process_dir(path, dirinfo->d_name,
578 : burp_mode, level+1))
579 : goto end;
580 0 : continue;
581 : }
582 9 : else if(!S_ISREG(info.st_mode)
583 9 : || !info.st_size) // ignore zero-length files
584 0 : continue;
585 :
586 9 : newfile.dev=info.st_dev;
587 9 : newfile.ino=info.st_ino;
588 9 : newfile.nlink=info.st_nlink;
589 9 : newfile.full_cksum=0;
590 9 : newfile.part_cksum=0;
591 9 : newfile.next=NULL;
592 :
593 9 : if((find=find_key(info.st_size)))
594 : {
595 : //printf("check %d: %s\n", info.st_size, newfile.path);
596 6 : if(check_files(find, &newfile, &info))
597 : goto end;
598 : }
599 : else
600 : {
601 : //printf("add: %s\n", newfile.path);
602 3 : if(add_key(info.st_size, &newfile))
603 : goto end;
604 : }
605 : }
606 : ret=0;
607 : end:
608 3 : if(dirp) closedir(dirp);
609 3 : free_w(&newfile.path);
610 3 : free_w(&path);
611 3 : return ret;
612 : }
613 :
614 0 : static void sighandler(__attribute__ ((unused)) int signum)
615 : {
616 0 : locks_release_and_free(&locklist);
617 0 : exit(1);
618 : }
619 :
620 0 : static int is_regular_file(const char *clientconfdir, const char *file)
621 : {
622 : struct stat statp;
623 0 : char *fullpath=NULL;
624 0 : if(!(fullpath=prepend_s(clientconfdir, file)))
625 : return 0;
626 0 : if(lstat(fullpath, &statp))
627 : {
628 0 : free_w(&fullpath);
629 0 : return 0;
630 : }
631 0 : free_w(&fullpath);
632 0 : return S_ISREG(statp.st_mode);
633 : }
634 :
635 : static int in_group(struct strlist *grouplist, const char *dedup_group)
636 : {
637 : struct strlist *g;
638 :
639 0 : for(g=grouplist; g; g=g->next)
640 0 : if(!strcmp(g->path, dedup_group)) return 1;
641 :
642 : return 0;
643 : }
644 :
645 0 : static int iterate_over_clients(struct conf **globalcs,
646 : struct strlist *grouplist)
647 : {
648 0 : int ret=0;
649 0 : DIR *dirp=NULL;
650 0 : struct conf **cconfs=NULL;
651 0 : struct dirent *dirinfo=NULL;
652 0 : const char *globalclientconfdir=get_string(globalcs[OPT_CLIENTCONFDIR]);
653 :
654 0 : if(!(cconfs=confs_alloc())) return -1;
655 0 : if(confs_init(cconfs)) return -1;
656 :
657 0 : if(!(dirp=opendir(globalclientconfdir)))
658 : {
659 0 : logp("Could not opendir '%s': %s\n",
660 0 : globalclientconfdir, strerror(errno));
661 0 : return 0;
662 : }
663 0 : while((dirinfo=readdir(dirp)))
664 : {
665 0 : char *lockfile=NULL;
666 0 : char *lockfilebase=NULL;
667 0 : char *client_lockdir=NULL;
668 0 : struct lock *lock=NULL;
669 :
670 0 : if(dirinfo->d_ino==0
671 : // looks_like...() also avoids '.' and '..'.
672 0 : || looks_like_tmp_or_hidden_file(dirinfo->d_name)
673 0 : || !is_regular_file(globalclientconfdir, dirinfo->d_name))
674 0 : continue;
675 :
676 0 : confs_free_content(cconfs);
677 0 : if(confs_init(cconfs)) return -1;
678 :
679 0 : if(set_string(cconfs[OPT_CNAME], dirinfo->d_name))
680 : return -1;
681 :
682 0 : if(conf_load_clientconfdir(globalcs, cconfs))
683 : {
684 0 : logp("could not load config for client %s\n",
685 : dirinfo->d_name);
686 0 : return 0;
687 : }
688 :
689 0 : if(grouplist)
690 : {
691 0 : const char *dedup_group=
692 0 : get_string(cconfs[OPT_DEDUP_GROUP]);
693 0 : if(!dedup_group
694 0 : || !in_group(grouplist, dedup_group))
695 0 : continue;
696 : }
697 :
698 0 : if(!(client_lockdir=get_string(cconfs[OPT_CLIENT_LOCKDIR])))
699 0 : client_lockdir=get_string(cconfs[OPT_DIRECTORY]);
700 :
701 0 : if(!(lockfilebase=prepend_s(client_lockdir, dirinfo->d_name))
702 0 : || !(lockfile=prepend_s(lockfilebase, BEDUP_LOCKFILE_NAME)))
703 : {
704 0 : free_w(&lockfilebase);
705 0 : free_w(&lockfile);
706 0 : ret=-1;
707 0 : break;
708 : }
709 0 : free_w(&lockfilebase);
710 :
711 0 : if(!(lock=lock_alloc_and_init(lockfile)))
712 : {
713 : ret=-1;
714 : break;
715 : }
716 0 : lock_get(lock);
717 0 : free_w(&lockfile);
718 :
719 0 : if(lock->status!=GET_LOCK_GOT)
720 : {
721 0 : logp("Could not get %s\n", lock->path);
722 0 : continue;
723 : }
724 0 : logp("Got %s\n", lock->path);
725 :
726 : // Remember that we got that lock.
727 0 : lock_add_to_list(&locklist, lock);
728 :
729 0 : switch(process_dir(get_string(cconfs[OPT_DIRECTORY]),
730 : dirinfo->d_name,
731 : 1 /* burp mode */, 0 /* level */))
732 : {
733 0 : case 0: ccount++;
734 0 : case 1: continue;
735 : default: ret=-1; break;
736 : }
737 : break;
738 : }
739 0 : closedir(dirp);
740 :
741 0 : locks_release_and_free(&locklist);
742 :
743 0 : confs_free(&cconfs);
744 :
745 0 : return ret;
746 : }
747 :
748 0 : static int process_from_conf(const char *configfile, char **groups)
749 : {
750 0 : int ret=-1;
751 0 : struct conf **globalcs=NULL;
752 0 : struct strlist *grouplist=NULL;
753 0 : struct lock *globallock=NULL;
754 :
755 0 : signal(SIGABRT, &sighandler);
756 0 : signal(SIGTERM, &sighandler);
757 0 : signal(SIGINT, &sighandler);
758 :
759 0 : if(*groups)
760 : {
761 0 : char *tok=NULL;
762 0 : if((tok=strtok(*groups, ",\n")))
763 : {
764 : do
765 : {
766 0 : if(strlist_add(&grouplist, tok, 1))
767 : {
768 0 : log_out_of_memory(__func__);
769 0 : goto end;
770 : }
771 0 : } while((tok=strtok(NULL, ",\n")));
772 : }
773 0 : if(!grouplist)
774 : {
775 0 : logp("unable to read list of groups\n");
776 0 : goto end;
777 : }
778 : }
779 :
780 : // Read directories from config files, and get locks.
781 0 : if(!(globalcs=confs_alloc())
782 0 : || confs_init(globalcs)
783 0 : || conf_load_global_only(configfile, globalcs))
784 : goto end;
785 :
786 0 : if(get_e_burp_mode(globalcs[OPT_BURP_MODE])!=BURP_MODE_SERVER)
787 : {
788 0 : logp("%s is not a server config file\n", configfile);
789 0 : goto end;
790 : }
791 0 : logp("Dedup clients from %s\n",
792 0 : get_string(globalcs[OPT_CLIENTCONFDIR]));
793 0 : maxlinks=get_int(globalcs[OPT_MAX_HARDLINKS]);
794 0 : if(grouplist)
795 : {
796 0 : struct strlist *g=NULL;
797 0 : logp("in dedup groups:\n");
798 0 : for(g=grouplist; g; g=g->next)
799 0 : logp("%s\n", g->path);
800 : }
801 : else
802 : {
803 0 : char *lockpath=NULL;
804 0 : const char *opt_lockfile=confs_get_lockfile(globalcs);
805 : // Only get the global lock when doing a global run.
806 : // If you are doing individual groups, you are likely
807 : // to want to do many different dedup jobs and a
808 : // global lock would get in the way.
809 0 : if(!(lockpath=prepend(opt_lockfile, ".bedup"))
810 0 : || !(globallock=lock_alloc_and_init(lockpath)))
811 : goto end;
812 0 : lock_get(globallock);
813 0 : if(globallock->status!=GET_LOCK_GOT)
814 : {
815 0 : logp("Could not get lock %s (%d)\n", lockpath,
816 : globallock->status);
817 0 : free_w(&lockpath);
818 0 : goto end;
819 : }
820 0 : logp("Got %s\n", lockpath);
821 : }
822 0 : ret=iterate_over_clients(globalcs, grouplist);
823 : end:
824 0 : confs_free(&globalcs);
825 0 : lock_release(globallock);
826 0 : lock_free(&globallock);
827 0 : strlists_free(&grouplist);
828 0 : return ret;
829 : }
830 :
831 3 : static int process_from_command_line(int argc, char *argv[])
832 : {
833 : int i;
834 6 : for(i=optind; i<argc; i++)
835 : {
836 : // Strip trailing slashes, for tidiness.
837 3 : if(argv[i][strlen(argv[i])-1]=='/')
838 0 : argv[i][strlen(argv[i])-1]='\0';
839 3 : if(process_dir("", argv[i],
840 : 0 /* not burp mode */, 0 /* level */))
841 : return 1;
842 : }
843 : return 0;
844 : }
845 :
846 2 : static int usage(void)
847 : {
848 2 : logfmt("\nUsage: %s [options]\n", prog);
849 2 : logfmt("\n");
850 2 : logfmt(" Options:\n");
851 2 : logfmt(" -c <path> Path to config file (default: %s).\n", config_default_path());
852 2 : logfmt(" -g <list of group names> Only run on the directories of clients that\n");
853 2 : logfmt(" are in one of the groups specified.\n");
854 2 : logfmt(" The list is comma-separated. To put a client in a\n");
855 2 : logfmt(" group, use the 'dedup_group' option in the client\n");
856 2 : logfmt(" configuration file on the server.\n");
857 2 : logfmt(" -h|-? Print this text and exit.\n");
858 2 : logfmt(" -d Delete any duplicate files found.\n");
859 2 : logfmt(" (non-%s mode only)\n", PACKAGE_TARNAME);
860 2 : logfmt(" -l Hard link any duplicate files found.\n");
861 2 : logfmt(" -m <number> Maximum number of hard links to a single file.\n");
862 2 : logfmt(" (non-%s mode only - in burp mode, use the\n", PACKAGE_TARNAME);
863 2 : logfmt(" max_hardlinks option in the configuration file)\n");
864 2 : logfmt(" The default is %d. On ext3, the maximum number\n", DEF_MAX_LINKS);
865 2 : logfmt(" of links possible is 32000, but space is needed\n");
866 2 : logfmt(" for the normal operation of %s.\n", PACKAGE_TARNAME);
867 2 : logfmt(" -n <list of directories> Non-%s mode. Deduplicate any (set of) directories.\n", PACKAGE_TARNAME);
868 2 : logfmt(" -v Print duplicate paths.\n");
869 2 : logfmt(" -V Print version and exit.\n");
870 2 : logfmt("\n");
871 2 : logfmt("By default, %s will read %s and deduplicate client storage\n", prog, config_default_path());
872 2 : logfmt("directories using special knowledge of the structure.\n");
873 2 : logfmt("\n");
874 2 : logfmt("With '-n', this knowledge is turned off and you have to specify the directories\n");
875 2 : logfmt("to deduplicate on the command line. Running with '-n' is therefore dangerous\n");
876 2 : logfmt("if you are deduplicating %s storage directories.\n\n", PACKAGE_TARNAME);
877 2 : return 1;
878 : }
879 :
880 14 : int run_bedup(int argc, char *argv[])
881 : {
882 14 : int ret=0;
883 14 : int option=0;
884 14 : int nonburp=0;
885 14 : char *groups=NULL;
886 14 : int givenconfigfile=0;
887 14 : const char *configfile=NULL;
888 :
889 14 : configfile=config_default_path();
890 14 : snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
891 :
892 46 : while((option=getopt(argc, argv, "c:dg:hlm:nvV?"))!=-1)
893 : {
894 21 : switch(option)
895 : {
896 : case 'c':
897 1 : configfile=optarg;
898 1 : givenconfigfile=1;
899 1 : break;
900 : case 'd':
901 2 : deletedups=1;
902 2 : break;
903 : case 'g':
904 1 : groups=optarg;
905 1 : break;
906 : case 'l':
907 3 : makelinks=1;
908 3 : break;
909 : case 'm':
910 6 : maxlinks=atoi(optarg);
911 3 : break;
912 : case 'n':
913 7 : nonburp=1;
914 7 : break;
915 : case 'V':
916 1 : logfmt("%s-%s\n", prog, PACKAGE_VERSION);
917 1 : return 0;
918 : case 'v':
919 1 : verbose=1;
920 1 : break;
921 : case 'h':
922 : case '?':
923 2 : return usage();
924 : }
925 : }
926 :
927 11 : if(nonburp && givenconfigfile)
928 : {
929 1 : logp("-n and -c options are mutually exclusive\n");
930 1 : return 1;
931 : }
932 10 : if(nonburp && groups)
933 : {
934 1 : logp("-n and -g options are mutually exclusive\n");
935 1 : return 1;
936 : }
937 9 : if(!nonburp && maxlinks!=DEF_MAX_LINKS)
938 : {
939 1 : logp("-m option is specified via the configuration file in %s mode (max_hardlinks=)\n", PACKAGE_TARNAME);
940 1 : return 1;
941 : }
942 8 : if(deletedups && makelinks)
943 : {
944 1 : logp("-d and -l options are mutually exclusive\n");
945 1 : return 1;
946 : }
947 7 : if(deletedups && !nonburp)
948 : {
949 1 : logp("-d option requires -n option\n");
950 1 : return 1;
951 : }
952 :
953 6 : if(optind>=argc)
954 : {
955 1 : if(nonburp)
956 : {
957 1 : logp("No directories found after options\n");
958 1 : return 1;
959 : }
960 : }
961 : else
962 : {
963 5 : if(!nonburp)
964 : {
965 1 : logp("Do not specify extra arguments.\n");
966 1 : return 1;
967 : }
968 : }
969 :
970 4 : if(maxlinks<2)
971 : {
972 1 : logp("The argument to -m needs to be greater than 1.\n");
973 1 : return 1;
974 : }
975 :
976 3 : if(nonburp)
977 : {
978 : // Read directories from command line.
979 3 : if(process_from_command_line(argc, argv))
980 0 : ret=1;
981 : }
982 : else
983 : {
984 0 : if(process_from_conf(configfile, &groups))
985 0 : ret=1;
986 : }
987 :
988 3 : if(!nonburp)
989 : {
990 0 : logp("%d client storages scanned\n", ccount);
991 : }
992 3 : logp("%" PRIu64 " duplicate %s found\n",
993 3 : count, count==1?"file":"files");
994 9 : logp("%" PRIu64 " bytes %s%s\n",
995 4 : savedbytes, (makelinks || deletedups)?"saved":"saveable",
996 : bytes_to_human(savedbytes));
997 3 : mystruct_delete_all();
998 3 : return ret;
999 : }
|
