Line data Source code
1 : #include "../../../burp.h"
2 : #include "../../../alloc.h"
3 : #include "../../../asfd.h"
4 : #include "../../../lock.h"
5 : #include "../../../log.h"
6 : #include "../../../prepend.h"
7 : #include "../../../protocol2/blist.h"
8 : #include "../../../protocol2/blk.h"
9 : #include "candidate.h"
10 : #include "champ_chooser.h"
11 : #include "hash.h"
12 : #include "incoming.h"
13 : #include "scores.h"
14 :
15 : static void try_lock_msg(int seconds)
16 : {
17 0 : logp("Unable to get sparse lock for %d seconds.\n", seconds);
18 : }
19 :
20 34 : static int try_to_get_lock(struct lock *lock)
21 : {
22 : // Sleeping for 1800*2 seconds makes 1 hour.
23 : // This should be super generous.
24 34 : int lock_tries=0;
25 34 : int lock_tries_max=1800;
26 34 : int sleeptime=2;
27 :
28 : while(1)
29 : {
30 34 : lock_get(lock);
31 34 : switch(lock->status)
32 : {
33 : case GET_LOCK_GOT:
34 34 : logp("Got sparse lock\n");
35 34 : return 0;
36 : case GET_LOCK_NOT_GOT:
37 0 : lock_tries++;
38 : if(lock_tries>lock_tries_max)
39 : {
40 : try_lock_msg(lock_tries_max*sleeptime);
41 : return -1;
42 : }
43 : // Log every 10 seconds.
44 : if(lock_tries%(10/sleeptime))
45 : {
46 0 : try_lock_msg(lock_tries_max*sleeptime);
47 0 : logp("Giving up.\n");
48 0 : return -1;
49 : }
50 : sleep(sleeptime);
51 : continue;
52 : case GET_LOCK_ERROR:
53 : default:
54 0 : logp("Unable to get global sparse lock.\n");
55 0 : return -1;
56 : }
57 : }
58 : // Never reached.
59 : return -1;
60 : }
61 :
62 34 : struct lock *try_to_get_sparse_lock(const char *sparse_path)
63 : {
64 34 : char *lockfile=NULL;
65 34 : struct lock *lock=NULL;
66 68 : if(!(lockfile=prepend_n(sparse_path, "lock", strlen("lock"), "."))
67 34 : || !(lock=lock_alloc_and_init(lockfile))
68 68 : || try_to_get_lock(lock))
69 0 : lock_free(&lock);
70 34 : free_w(&lockfile);
71 34 : return lock;
72 : }
73 :
74 0 : static int load_existing_sparse(const char *datadir, struct scores *scores)
75 : {
76 0 : int ret=-1;
77 : struct stat statp;
78 0 : struct lock *lock=NULL;
79 0 : char *sparse_path=NULL;
80 0 : if(!(sparse_path=prepend_s(datadir, "sparse"))) goto end;
81 : // Best not let other things mess with the sparse lock while we are
82 : // trying to read it.
83 0 : if(!(lock=try_to_get_sparse_lock(sparse_path)))
84 : goto end;
85 0 : if(lstat(sparse_path, &statp))
86 : {
87 : ret=0;
88 : goto end;
89 : }
90 0 : if(candidate_load(NULL, sparse_path, scores))
91 : goto end;
92 0 : ret=0;
93 : end:
94 0 : free_w(&sparse_path);
95 0 : lock_release(lock);
96 0 : lock_free(&lock);
97 0 : return ret;
98 : }
99 :
100 0 : struct scores *champ_chooser_init(const char *datadir)
101 : {
102 0 : struct scores *scores=NULL;
103 0 : if(!(scores=scores_alloc())
104 0 : || load_existing_sparse(datadir, scores))
105 : goto error;
106 0 : return scores;
107 : error:
108 0 : scores_free(&scores);
109 0 : return NULL;
110 : }
111 :
112 4096 : static int already_got_block(struct asfd *asfd, struct blk *blk)
113 : {
114 : static struct hash_weak *hash_weak;
115 :
116 : // If already got, need to overwrite the references.
117 4096 : if((hash_weak=hash_weak_find(blk->fingerprint)))
118 : {
119 : static struct hash_strong *hash_strong;
120 0 : if((hash_strong=hash_strong_find(
121 0 : hash_weak, blk->md5sum)))
122 : {
123 0 : blk->savepath=hash_strong->savepath;
124 : //printf("FOUND: %s %s\n", blk->weak, blk->strong);
125 : //printf("F");
126 0 : blk->got=BLK_GOT;
127 0 : asfd->in->got++;
128 : return 0;
129 : }
130 : else
131 : {
132 : // printf("COLLISION: %s %s\n", blk->weak, blk->strong);
133 : // collisions++;
134 : }
135 : }
136 :
137 4096 : blk->got=BLK_NOT_GOT;
138 : //printf(".");
139 : return 0;
140 : }
141 :
142 : #define CHAMPS_MAX 10
143 :
144 1 : int deduplicate(struct asfd *asfd, const char *directory, struct scores *scores)
145 : {
146 : struct blk *blk;
147 1 : struct incoming *in=asfd->in;
148 : struct candidate *champ;
149 1 : struct candidate *champ_last=NULL;
150 1 : int count=0;
151 1 : int blk_count=0;
152 :
153 1 : if(!in) return 0;
154 :
155 1 : incoming_found_reset(in);
156 1 : count=0;
157 2 : while(count!=CHAMPS_MAX
158 1 : && (champ=candidates_choose_champ(in, champ_last, scores)))
159 : {
160 : // printf("Got champ: %s %d\n", champ->path, *(champ->score));
161 0 : switch(hash_load(champ->path, directory))
162 : {
163 : case HASH_RET_OK:
164 0 : count++;
165 0 : champ_last=champ;
166 0 : break;
167 : case HASH_RET_PERM:
168 : return -1;
169 : case HASH_RET_TEMP:
170 0 : champ->deleted=1;
171 0 : break;
172 : }
173 : }
174 :
175 1 : blk_count=0;
176 4097 : for(blk=asfd->blist->blk_to_dedup; blk; blk=blk->next)
177 : {
178 : //printf("try: %lu\n", blk->index);
179 4096 : blk_count++;
180 :
181 4096 : if(blk_is_zero_length(blk))
182 : {
183 : //printf("got: %s %s\n", blk->weak, blk->strong);
184 0 : blk->got=BLK_GOT;
185 0 : in->got++;
186 0 : continue;
187 : }
188 :
189 : // If already got, this function will set blk->save_path
190 : // to be the location of the already got block.
191 4096 : if(already_got_block(asfd, blk)) return -1;
192 :
193 : //printf("after agb: %lu %d\n", blk->index, blk->got);
194 : }
195 :
196 : logp("%s: %04d/%04zu - %04d/%04d\n",
197 1 : asfd->desc, count, candidates_len, in->got, blk_count);
198 :
199 : // Start the incoming array again.
200 1 : in->size=0;
201 : // Destroy the deduplication hash table.
202 1 : hash_delete_all();
203 :
204 1 : asfd->blist->blk_to_dedup=NULL;
205 :
206 1 : return 0;
207 : }
|
