Line data Source code
1 : #include "../../../burp.h"
2 : #include "../../../alloc.h"
3 : #include "../../../asfd.h"
4 : #include "../../../lock.h"
5 : #include "../../../log.h"
6 : #include "../../../prepend.h"
7 : #include "../../../protocol2/blist.h"
8 : #include "../../../protocol2/blk.h"
9 : #include "candidate.h"
10 : #include "champ_chooser.h"
11 : #include "hash.h"
12 : #include "incoming.h"
13 : #include "scores.h"
14 :
15 0 : static void try_lock_msg(int seconds)
16 : {
17 0 : logp("Unable to get sparse lock for %d seconds.\n", seconds);
18 0 : }
19 :
20 32 : static int try_to_get_lock(struct lock *lock)
21 : {
22 : // Sleeping for 1800*2 seconds makes 1 hour.
23 : // This should be super generous.
24 32 : int lock_tries=0;
25 32 : int lock_tries_max=1800;
26 32 : int sleeptime=2;
27 :
28 : while(1)
29 : {
30 32 : lock_get(lock);
31 32 : switch(lock->status)
32 : {
33 : case GET_LOCK_GOT:
34 32 : logp("Got sparse lock\n");
35 32 : return 0;
36 : case GET_LOCK_NOT_GOT:
37 0 : lock_tries++;
38 0 : if(lock_tries>lock_tries_max)
39 : {
40 0 : try_lock_msg(lock_tries_max*sleeptime);
41 0 : return -1;
42 : }
43 : // Log every 10 seconds.
44 0 : if(lock_tries%(10/sleeptime))
45 : {
46 0 : try_lock_msg(lock_tries_max*sleeptime);
47 0 : logp("Giving up.\n");
48 0 : return -1;
49 : }
50 0 : sleep(sleeptime);
51 0 : continue;
52 : case GET_LOCK_ERROR:
53 : default:
54 0 : logp("Unable to get global sparse lock.\n");
55 0 : return -1;
56 : }
57 : }
58 : // Never reached.
59 0 : return -1;
60 : }
61 :
62 32 : struct lock *try_to_get_sparse_lock(const char *sparse_path)
63 : {
64 32 : char *lockfile=NULL;
65 32 : struct lock *lock=NULL;
66 96 : if(!(lockfile=prepend_n(sparse_path, "lock", strlen("lock"), "."))
67 32 : || !(lock=lock_alloc_and_init(lockfile))
68 96 : || try_to_get_lock(lock))
69 0 : lock_free(&lock);
70 32 : free_w(&lockfile);
71 32 : return lock;
72 : }
73 :
74 0 : static int load_existing_sparse(const char *datadir, struct scores *scores)
75 : {
76 0 : int ret=-1;
77 : struct stat statp;
78 0 : struct lock *lock=NULL;
79 0 : char *sparse_path=NULL;
80 0 : if(!(sparse_path=prepend_s(datadir, "sparse"))) goto end;
81 : // Best not let other things mess with the sparse lock while we are
82 : // trying to read it.
83 0 : if(!(lock=try_to_get_sparse_lock(sparse_path)))
84 0 : goto end;
85 0 : if(lstat(sparse_path, &statp))
86 : {
87 0 : ret=0;
88 0 : goto end;
89 : }
90 0 : if(candidate_load(NULL, sparse_path, scores))
91 0 : goto end;
92 0 : ret=0;
93 : end:
94 0 : free_w(&sparse_path);
95 0 : lock_release(lock);
96 0 : lock_free(&lock);
97 0 : return ret;
98 : }
99 :
100 0 : struct scores *champ_chooser_init(const char *datadir)
101 : {
102 0 : struct scores *scores=NULL;
103 0 : if(!(scores=scores_alloc())
104 0 : || load_existing_sparse(datadir, scores))
105 0 : goto error;
106 0 : return scores;
107 : error:
108 0 : scores_free(&scores);
109 0 : return NULL;
110 : }
111 :
112 0 : static int already_got_block(struct asfd *asfd, struct blk *blk)
113 : {
114 : static struct hash_weak *hash_weak;
115 :
116 : // If already got, need to overwrite the references.
117 0 : if((hash_weak=hash_weak_find(blk->fingerprint)))
118 : {
119 : static struct hash_strong *hash_strong;
120 0 : if((hash_strong=hash_strong_find(
121 0 : hash_weak, blk->md5sum)))
122 : {
123 0 : blk->savepath=hash_strong->savepath;
124 : //printf("FOUND: %s %s\n", blk->weak, blk->strong);
125 : //printf("F");
126 0 : blk->got=BLK_GOT;
127 0 : asfd->in->got++;
128 0 : return 0;
129 : }
130 : else
131 : {
132 : // printf("COLLISION: %s %s\n", blk->weak, blk->strong);
133 : // collisions++;
134 : }
135 : }
136 :
137 0 : blk->got=BLK_NOT_GOT;
138 : //printf(".");
139 0 : return 0;
140 : }
141 :
142 : #define CHAMPS_MAX 10
143 :
144 0 : int deduplicate(struct asfd *asfd, const char *directory, struct scores *scores)
145 : {
146 : struct blk *blk;
147 0 : struct incoming *in=asfd->in;
148 : struct candidate *champ;
149 0 : struct candidate *champ_last=NULL;
150 0 : int count=0;
151 0 : int blk_count=0;
152 :
153 0 : if(!in) return 0;
154 :
155 0 : incoming_found_reset(in);
156 0 : count=0;
157 0 : while(count!=CHAMPS_MAX
158 0 : && (champ=candidates_choose_champ(in, champ_last, scores)))
159 : {
160 : // printf("Got champ: %s %d\n", champ->path, *(champ->score));
161 0 : switch(hash_load(champ->path, directory))
162 : {
163 : case HASH_RET_OK:
164 0 : count++;
165 0 : champ_last=champ;
166 0 : break;
167 : case HASH_RET_PERM:
168 0 : return -1;
169 : case HASH_RET_TEMP:
170 0 : champ->deleted=1;
171 0 : break;
172 : }
173 : }
174 :
175 0 : blk_count=0;
176 0 : for(blk=asfd->blist->blk_to_dedup; blk; blk=blk->next)
177 : {
178 : //printf("try: %lu\n", blk->index);
179 0 : blk_count++;
180 :
181 0 : if(blk_is_zero_length(blk))
182 : {
183 : //printf("got: %s %s\n", blk->weak, blk->strong);
184 0 : blk->got=BLK_GOT;
185 0 : in->got++;
186 0 : continue;
187 : }
188 :
189 : // If already got, this function will set blk->save_path
190 : // to be the location of the already got block.
191 0 : if(already_got_block(asfd, blk)) return -1;
192 :
193 : //printf("after agb: %lu %d\n", blk->index, blk->got);
194 : }
195 :
196 : logp("%s: %04d/%04d - %04d/%04d\n",
197 0 : asfd->desc, count, candidates_len, in->got, blk_count);
198 :
199 : // Start the incoming array again.
200 0 : in->size=0;
201 : // Destroy the deduplication hash table.
202 0 : hash_delete_all();
203 :
204 0 : asfd->blist->blk_to_dedup=NULL;
205 :
206 0 : return 0;
207 : }
|
