Line data Source code
1 : #include "../../burp.h"
2 : #include "../../asfd.h"
3 : #include "../../async.h"
4 : #include "../../attribs.h"
5 : #include "../../base64.h"
6 : #include "../../cmd.h"
7 : #include "../../cntr.h"
8 : #include "../../handy.h"
9 : #include "../../hexmap.h"
10 : #include "../../iobuf.h"
11 : #include "../../log.h"
12 : #include "../../server/manio.h"
13 : #include "../../protocol2/blist.h"
14 : #include "../../slist.h"
15 : #include "../manios.h"
16 : #include "../resume.h"
17 : #include "champ_chooser/champ_server.h"
18 : #include "dpth.h"
19 :
20 : #define END_SIGS 0x01
21 : #define END_BACKUP 0x02
22 : #define END_REQUESTS 0x04
23 : #define END_BLK_REQUESTS 0x08
24 :
25 : static int breaking=0;
26 : static int breakcount=0;
27 :
28 0 : static int data_needed(struct sbuf *sb)
29 : {
30 0 : if(sb->path.cmd==CMD_FILE) return 1;
31 0 : return 0;
32 : }
33 :
34 0 : static int manio_component_to_chfd(struct asfd *chfd, char *path)
35 : {
36 : struct iobuf wbuf;
37 0 : iobuf_from_str(&wbuf, CMD_MANIFEST, path);
38 0 : return chfd->write(chfd, &wbuf);
39 : }
40 :
41 0 : static int unchanged(struct sbuf *csb, struct sbuf *sb,
42 : struct blk **blk, struct manios *manios, struct asfd *chfd)
43 : {
44 0 : int ret=-1;
45 0 : char *fpath=NULL;
46 0 : if(!(fpath=strdup_w(manios->changed->offset->fpath, __func__)))
47 0 : goto end;
48 0 : if(manio_copy_entry(csb, sb, blk,
49 0 : manios->current, manios->unchanged)<0)
50 0 : goto end;
51 0 : if(strcmp(fpath, manios->changed->offset->fpath))
52 : {
53 : // If the copy crossed a manio boundary, we should tell the
54 : // champ server to load the previous one as a candidate.
55 0 : if(manio_component_to_chfd(chfd, fpath))
56 0 : goto end;
57 : }
58 0 : ret=0;
59 : end:
60 0 : free_w(&fpath);
61 0 : return ret;
62 : }
63 :
64 : // Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
65 0 : static int found_in_current_manifest(struct sbuf *csb, struct sbuf *sb,
66 : struct manios *manios, struct blk **blk, struct asfd *chfd)
67 : {
68 : // Located the entry in the current manifest.
69 : // If the file type changed, I think it is time to back it up again
70 : // (for example, EFS changing to normal file, or back again).
71 0 : if(csb->path.cmd!=sb->path.cmd)
72 : {
73 0 : if(manio_forward_through_sigs(csb, blk, manios->current)<0)
74 0 : return -1;
75 0 : return 1;
76 : }
77 :
78 : // mtime is the actual file data.
79 : // ctime is the attributes or meta data.
80 0 : if(csb->statp.st_mtime==sb->statp.st_mtime
81 0 : && csb->statp.st_ctime==sb->statp.st_ctime)
82 : {
83 : // Got an unchanged file.
84 0 : return unchanged(csb, sb, blk, manios, chfd);
85 : }
86 :
87 0 : if(csb->statp.st_mtime==sb->statp.st_mtime
88 0 : && csb->statp.st_ctime!=sb->statp.st_ctime)
89 : {
90 : // FIX THIS:
91 : // File data stayed the same, but attributes or meta data
92 : // changed. We already have the attributes, but may need to
93 : // get extra meta data.
94 0 : return unchanged(csb, sb, blk, manios, chfd);
95 : }
96 :
97 : // File data changed.
98 0 : if(manio_forward_through_sigs(csb, blk, manios->current)<0)
99 0 : return -1;
100 0 : return 1;
101 : }
102 :
103 : // Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
104 0 : static int entry_changed(struct sbuf *sb,
105 : struct manios *manios, struct asfd *chfd)
106 : {
107 : static int finished=0;
108 : static struct sbuf *csb=NULL;
109 : static struct blk *blk=NULL;
110 :
111 0 : if(finished) return 1;
112 :
113 0 : if(!csb && !(csb=sbuf_alloc(PROTO_2))) return -1;
114 :
115 0 : if(csb->path.buf)
116 : {
117 : // Already have an entry.
118 : }
119 : else
120 : {
121 : // Need to read another.
122 0 : if(!blk && !(blk=blk_alloc())) return -1;
123 0 : switch(manio_read_with_blk(manios->current,
124 0 : csb, blk, NULL))
125 : {
126 : case 1: // Reached the end.
127 0 : sbuf_free(&csb);
128 0 : blk_free(&blk);
129 0 : finished=1;
130 0 : return 1;
131 0 : case -1: return -1;
132 : }
133 0 : if(!csb->path.buf)
134 : {
135 0 : logp("Should have a path at this point, but do not, in %s\n", __func__);
136 0 : return -1;
137 : }
138 : // Got an entry.
139 : }
140 :
141 : while(1)
142 : {
143 0 : switch(sbuf_pathcmp(csb, sb))
144 : {
145 : case 0: return found_in_current_manifest(csb, sb,
146 0 : manios, &blk, chfd);
147 0 : case 1: return 1;
148 : case -1:
149 : // Behind - need to read more data from the old
150 : // manifest.
151 0 : switch(manio_read_with_blk(manios->current,
152 0 : csb, blk, NULL))
153 : {
154 : case 1: // Reached the end.
155 0 : sbuf_free(&csb);
156 0 : blk_free(&blk);
157 0 : return 1;
158 0 : case -1: return -1;
159 : }
160 : // Got something, go back around the loop.
161 : }
162 : }
163 :
164 0 : return 0;
165 : }
166 :
167 0 : static int add_data_to_store(struct cntr *cntr,
168 : struct slist *slist, struct iobuf *rbuf, struct dpth *dpth)
169 : {
170 : static struct blk *blk=NULL;
171 :
172 : // Find the first one in the list that was requested.
173 : // FIX THIS: Going up the list here, and then later
174 : // when writing to the manifest is not efficient.
175 0 : for(blk=slist->blist->head;
176 0 : blk && (!blk->requested || blk->got==BLK_GOT); blk=blk->next)
177 : {
178 : // logp("try: %d %d\n", blk->index, blk->got);
179 : }
180 0 : if(!blk)
181 : {
182 0 : logp("Received data but could not find next requested block.\n");
183 0 : if(!slist->blist->head)
184 0 : logp("and slist->blist->head is null\n");
185 : else
186 0 : logp("head index: %d\n", slist->blist->head->index);
187 0 : return -1;
188 : }
189 :
190 : // Add it to the data store straight away.
191 0 : if(dpth_protocol2_fwrite(dpth, rbuf, blk)) return -1;
192 :
193 0 : cntr_add(cntr, CMD_DATA, 0);
194 0 : cntr_add_recvbytes(cntr, blk->length);
195 :
196 0 : blk->got=BLK_GOT;
197 0 : blk=blk->next;
198 :
199 0 : return 0;
200 : }
201 :
202 0 : static int set_up_for_sig_info(struct slist *slist, struct sbuf *inew)
203 : {
204 : struct sbuf *sb;
205 :
206 0 : for(sb=slist->add_sigs_here; sb; sb=sb->next)
207 : {
208 0 : if(!sb->protocol2->index) continue;
209 0 : if(inew->protocol2->index==sb->protocol2->index) break;
210 : }
211 0 : if(!sb)
212 : {
213 : logp("Could not find %lu in request list %d\n",
214 0 : inew->protocol2->index, sb->protocol2->index);
215 0 : return -1;
216 : }
217 : // Replace the attribs with the more recent values.
218 0 : iobuf_free_content(&sb->attr);
219 0 : iobuf_move(&sb->attr, &inew->attr);
220 :
221 : // Mark the end of the previous file.
222 0 : slist->add_sigs_here->protocol2->bend=slist->blist->tail;
223 :
224 0 : slist->add_sigs_here=sb;
225 :
226 : // Incoming sigs now need to get added to 'add_sigs_here'
227 0 : return 0;
228 : }
229 :
230 : /*
231 : static void dump_blks(const char *msg, struct blk *b)
232 : {
233 : struct blk *xx;
234 : for(xx=b; xx; xx=xx->next)
235 : printf("%s: %d %d %p\n", msg, xx->index, xx->got, xx);
236 : }
237 : */
238 :
239 0 : static int add_to_sig_list(struct slist *slist, struct iobuf *rbuf)
240 : {
241 : // Goes on slist->add_sigs_here
242 : struct blk *blk;
243 : struct protocol2 *protocol2;
244 :
245 0 : if(!(blk=blk_alloc())) return -1;
246 0 : blist_add_blk(slist->blist, blk);
247 :
248 0 : protocol2=slist->add_sigs_here->protocol2;
249 0 : if(!protocol2->bstart) protocol2->bstart=blk;
250 0 : if(!protocol2->bsighead) protocol2->bsighead=blk;
251 :
252 0 : if(blk_set_from_iobuf_sig(blk, rbuf)) return -1;
253 :
254 : // Need to send sigs to champ chooser, therefore need to point
255 : // to the oldest unsent one if nothing is pointed to yet.
256 0 : if(!slist->blist->blk_for_champ_chooser)
257 0 : slist->blist->blk_for_champ_chooser=blk;
258 :
259 0 : return 0;
260 : }
261 :
262 0 : static int deal_with_read(struct iobuf *rbuf, struct slist *slist,
263 : struct cntr *cntr, uint8_t *end_flags, struct dpth *dpth)
264 : {
265 0 : int ret=0;
266 : static struct sbuf *inew=NULL;
267 :
268 0 : if(!inew && !(inew=sbuf_alloc(PROTO_2))) goto error;
269 :
270 0 : switch(rbuf->cmd)
271 : {
272 : /* Incoming block data. */
273 : case CMD_DATA:
274 0 : if(add_data_to_store(cntr, slist, rbuf, dpth))
275 0 : goto error;
276 0 : goto end;
277 :
278 : /* Incoming block signatures. */
279 : case CMD_ATTRIBS_SIGS:
280 : // New set of stuff incoming. Clean up.
281 0 : iobuf_free_content(&inew->attr);
282 0 : iobuf_move(&inew->attr, rbuf);
283 0 : inew->protocol2->index=decode_file_no(&inew->attr);
284 :
285 : // Need to go through slist to find the matching
286 : // entry.
287 0 : if(set_up_for_sig_info(slist, inew))
288 0 : goto error;
289 0 : return 0;
290 : case CMD_SIG:
291 0 : if(add_to_sig_list(slist, rbuf))
292 0 : goto error;
293 0 : goto end;
294 :
295 : /* Incoming control/message stuff. */
296 : case CMD_MESSAGE:
297 : case CMD_WARNING:
298 : {
299 0 : struct cntr *cntr=NULL;
300 0 : log_recvd(rbuf, cntr, 0);
301 0 : goto end;
302 : }
303 : case CMD_GEN:
304 0 : if(!strcmp(rbuf->buf, "sigs_end"))
305 : {
306 0 : (*end_flags)|=END_SIGS;
307 0 : goto end;
308 : }
309 0 : else if(!strcmp(rbuf->buf, "backup_end"))
310 : {
311 0 : (*end_flags)|=END_BACKUP;
312 0 : goto end;
313 : }
314 0 : break;
315 : default:
316 0 : break;
317 : }
318 :
319 0 : iobuf_log_unexpected(rbuf, __func__);
320 : error:
321 0 : ret=-1;
322 0 : sbuf_free(&inew);
323 : end:
324 0 : iobuf_free_content(rbuf);
325 0 : return ret;
326 : }
327 :
328 0 : static int encode_req(struct blk *blk, char *req)
329 : {
330 0 : char *p=req;
331 0 : p+=to_base64(blk->index, p);
332 0 : *p=0;
333 0 : return 0;
334 : }
335 :
336 0 : static int get_wbuf_from_sigs(struct iobuf *wbuf, struct slist *slist,
337 : uint8_t *end_flags)
338 : {
339 : static char req[32]="";
340 0 : struct sbuf *sb=slist->blks_to_request;
341 :
342 0 : while(sb && !(sb->flags & SBUF_NEED_DATA)) sb=sb->next;
343 :
344 0 : if(!sb)
345 : {
346 0 : slist->blks_to_request=NULL;
347 0 : if((*end_flags)&END_SIGS && !((*end_flags)&END_BLK_REQUESTS))
348 : {
349 : iobuf_from_str(wbuf,
350 0 : CMD_GEN, (char *)"blk_requests_end");
351 0 : (*end_flags)|=END_BLK_REQUESTS;
352 : }
353 0 : return 0;
354 : }
355 0 : if(!sb->protocol2->bsighead)
356 : {
357 : // Trying to move onto the next file.
358 : // ??? Does this really work?
359 0 : if(sb->protocol2->bend)
360 : {
361 0 : slist->blks_to_request=sb->next;
362 0 : printf("move to next\n");
363 : }
364 0 : if((*end_flags)&END_SIGS && !((*end_flags)&END_BLK_REQUESTS))
365 : {
366 : iobuf_from_str(wbuf,
367 0 : CMD_GEN, (char *)"blk_requests_end");
368 0 : (*end_flags)|=END_BLK_REQUESTS;
369 : }
370 0 : return 0;
371 : }
372 :
373 0 : if(sb->protocol2->bsighead->got==BLK_INCOMING)
374 0 : return 0;
375 :
376 0 : if(sb->protocol2->bsighead->got==BLK_NOT_GOT)
377 : {
378 0 : encode_req(sb->protocol2->bsighead, req);
379 0 : iobuf_from_str(wbuf, CMD_DATA_REQ, req);
380 0 : sb->protocol2->bsighead->requested=1;
381 : }
382 :
383 : // Move on.
384 0 : if(sb->protocol2->bsighead==sb->protocol2->bend)
385 : {
386 0 : slist->blks_to_request=sb->next;
387 0 : sb->protocol2->bsighead=sb->protocol2->bstart;
388 : }
389 : else
390 : {
391 0 : sb->protocol2->bsighead=sb->protocol2->bsighead->next;
392 : }
393 0 : return 0;
394 : }
395 :
396 0 : static void get_wbuf_from_files(struct iobuf *wbuf, struct slist *slist,
397 : struct manios *manios, uint8_t *end_flags)
398 : {
399 : static uint64_t file_no=1;
400 0 : struct sbuf *sb=slist->last_requested;
401 0 : if(!sb)
402 : {
403 0 : if(!manios->phase1 && !((*end_flags)&END_REQUESTS))
404 : {
405 0 : iobuf_from_str(wbuf, CMD_GEN, (char *)"requests_end");
406 0 : (*end_flags)|=END_REQUESTS;
407 : }
408 0 : return;
409 : }
410 :
411 0 : if(sb->flags & SBUF_SENT_PATH || !(sb->flags & SBUF_NEED_DATA))
412 : {
413 0 : slist->last_requested=sb->next;
414 0 : return;
415 : }
416 :
417 : // Only need to request the path at this stage.
418 0 : iobuf_copy(wbuf, &sb->path);
419 0 : sb->flags |= SBUF_SENT_PATH;
420 0 : sb->protocol2->index=file_no++;
421 : }
422 :
423 0 : static void sanity_before_sbuf_free(struct slist *slist, struct sbuf *sb)
424 : {
425 : // It is possible for the markers to drop behind.
426 0 : if(slist->tail==sb) slist->tail=sb->next;
427 0 : if(slist->last_requested==sb) slist->last_requested=sb->next;
428 0 : if(slist->add_sigs_here==sb) slist->add_sigs_here=sb->next;
429 0 : if(slist->blks_to_request==sb) slist->blks_to_request=sb->next;
430 0 : }
431 :
432 0 : static void get_wbuf_from_index(struct iobuf *wbuf, uint64_t index)
433 : {
434 : static char *p;
435 : static char tmp[32];
436 : //printf("%s: %d\n", __func__, index);
437 0 : p=tmp;
438 0 : p+=to_base64(index, tmp);
439 0 : *p='\0';
440 0 : iobuf_from_str(wbuf, CMD_WRAP_UP, tmp);
441 0 : }
442 :
443 0 : static int write_endfile(struct sbuf *sb, struct manios *manios)
444 : {
445 : struct iobuf endfile;
446 :
447 0 : if(sb->flags & SBUF_END_WRITTEN_TO_MANIFEST)
448 0 : return 0;
449 0 : if(!iobuf_is_filedata(&sb->path))
450 0 : return 0;
451 :
452 0 : sb->flags |= SBUF_END_WRITTEN_TO_MANIFEST;
453 : // FIX THIS: Should give a proper length and md5sum.
454 0 : iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0");
455 0 : return iobuf_send_msg_fzp(&endfile, manios->changed->fzp);
456 : }
457 :
458 0 : static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
459 : struct asfd *chfd, struct manios *manios,
460 : struct slist *slist, int end_flags)
461 : {
462 0 : int ret=-1;
463 : struct blk *blk;
464 : static struct iobuf wbuf;
465 0 : struct blist *blist=slist->blist;
466 :
467 0 : if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
468 : {
469 0 : if(manio_write_sbuf(manios->changed, sb)) goto end;
470 0 : sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
471 : }
472 :
473 0 : while((blk=sb->protocol2->bstart)
474 0 : && blk->got==BLK_GOT
475 0 : && (blk->next || end_flags&END_BACKUP))
476 : {
477 0 : if(blk->got_save_path
478 0 : && !blk_is_zero_length(blk))
479 : {
480 0 : if(breaking && breakcount--==0)
481 : {
482 0 : breakpoint(breaking, __func__);
483 0 : goto end;
484 : }
485 0 : if(manio_write_sig_and_path(manios->changed, blk))
486 0 : goto end;
487 0 : if(manios->changed->sig_count==0)
488 : {
489 : // Have finished a manifest file. Want to start
490 : // using it as a dedup candidate now.
491 0 : if(manio_component_to_chfd(chfd,
492 0 : manios->changed->offset->ppath))
493 0 : goto end;
494 :
495 0 : if(!blk->requested)
496 : {
497 : // Also let the client know, so that it
498 : // can free memory if there was a long
499 : // consecutive number of unrequested
500 : // blocks.
501 0 : get_wbuf_from_index(&wbuf, blk->index);
502 0 : if(asfd->write(asfd, &wbuf)) goto end;
503 : }
504 : }
505 : }
506 :
507 0 : if(blk==sb->protocol2->bend)
508 : {
509 0 : slist->head=sb->next;
510 0 : if(!(blist->head=sb->protocol2->bstart))
511 0 : blist->tail=NULL;
512 0 : sanity_before_sbuf_free(slist, sb);
513 0 : if(write_endfile(sb, manios)) return -1;
514 0 : sbuf_free(&sb);
515 0 : return 1;
516 : }
517 :
518 0 : if(sb->protocol2->bsighead==sb->protocol2->bstart)
519 0 : sb->protocol2->bsighead=blk->next;
520 0 : sb->protocol2->bstart=blk->next;
521 0 : if(blk==blist->blk_from_champ_chooser)
522 0 : blist->blk_from_champ_chooser=blk->next;
523 :
524 0 : blk_free(&blk);
525 : }
526 0 : if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
527 : {
528 : // Write endfile for the very last file.
529 0 : if(write_endfile(sb, manios)) return -1;
530 : }
531 0 : ret=0;
532 : end:
533 0 : if(!(blist->head=sb->protocol2->bstart)) blist->tail=NULL;
534 0 : return ret;
535 : }
536 :
537 0 : static int write_to_changed_file(struct asfd *asfd,
538 : struct asfd *chfd, struct manios *manios,
539 : struct slist *slist, int end_flags)
540 : {
541 : struct sbuf *sb;
542 0 : if(!slist) return 0;
543 :
544 0 : while((sb=slist->head))
545 : {
546 0 : if(sb->flags & SBUF_NEED_DATA)
547 : {
548 0 : switch(sbuf_needs_data(sb, asfd, chfd, manios, slist,
549 0 : end_flags))
550 : {
551 0 : case 0: return 0;
552 0 : case 1: continue;
553 0 : default: return -1;
554 : }
555 :
556 : }
557 : else
558 : {
559 : // No change, can go straight in.
560 0 : if(manio_write_sbuf(manios->changed, sb)) return -1;
561 0 : if(write_endfile(sb, manios)) return -1;
562 :
563 : // Move along.
564 0 : slist->head=sb->next;
565 :
566 0 : sanity_before_sbuf_free(slist, sb);
567 0 : sbuf_free(&sb);
568 : }
569 : }
570 0 : return 0;
571 : }
572 :
573 0 : static int maybe_add_from_scan(struct manios *manios,
574 : struct slist *slist, struct asfd *chfd)
575 : {
576 0 : int ret=-1;
577 0 : struct sbuf *snew=NULL;
578 :
579 : while(1)
580 : {
581 0 : if(!manios->phase1) return 0;
582 : // Limit the amount loaded into memory at any one time.
583 0 : if(slist && slist->head)
584 : {
585 0 : if(slist->head->protocol2->index
586 0 : - slist->tail->protocol2->index>4096)
587 0 : return 0;
588 : }
589 0 : if(!(snew=sbuf_alloc(PROTO_2))) goto end;
590 :
591 0 : switch(manio_read(manios->phase1, snew))
592 : {
593 0 : case 0: break;
594 0 : case 1: manio_close(&manios->phase1);
595 0 : return 0; // Finished.
596 0 : default: goto end;
597 : }
598 :
599 0 : switch(entry_changed(snew, manios, chfd))
600 : {
601 0 : case 0: continue; // No change, add to slist.
602 0 : case 1: break;
603 0 : default: goto end; // Error.
604 : }
605 :
606 0 : if(data_needed(snew)) snew->flags|=SBUF_NEED_DATA;
607 :
608 0 : slist_add_sbuf(slist, snew);
609 : }
610 : return 0;
611 : end:
612 0 : sbuf_free(&snew);
613 0 : return ret;
614 : }
615 :
616 0 : static int append_for_champ_chooser(struct asfd *chfd,
617 : struct blist *blist, int end_flags)
618 : {
619 : static int finished_sending=0;
620 : static struct iobuf wbuf;
621 : static struct blk *blk=NULL;
622 :
623 0 : while(blist->blk_for_champ_chooser)
624 : {
625 0 : blk=blist->blk_for_champ_chooser;
626 : // If we send too many blocks to the champ chooser at once,
627 : // it can go faster than we can send paths to completed
628 : // manifests to it. This means that deduplication efficiency
629 : // is reduced (although speed may be faster).
630 : // So limit the sending.
631 0 : if(blk->index
632 0 : - blist->head->index > MANIFEST_SIG_MAX)
633 0 : return 0;
634 :
635 0 : blk_to_iobuf_sig(blk, &wbuf);
636 :
637 0 : switch(chfd->append_all_to_write_buffer(chfd, &wbuf))
638 : {
639 0 : case APPEND_OK: break;
640 : case APPEND_BLOCKED:
641 0 : return 0; // Try again later.
642 0 : default: return -1;
643 : }
644 0 : blist->blk_for_champ_chooser=blk->next;
645 : }
646 0 : if(end_flags&END_SIGS
647 0 : && !finished_sending && !blist->blk_for_champ_chooser)
648 : {
649 0 : iobuf_from_str(&wbuf, CMD_GEN, (char *)"sigs_end");
650 0 : switch(chfd->append_all_to_write_buffer(chfd, &wbuf))
651 : {
652 0 : case APPEND_OK: break;
653 : case APPEND_BLOCKED:
654 0 : return 0; // Try again later.
655 0 : default: return -1;
656 : }
657 0 : finished_sending++;
658 : }
659 0 : return 0;
660 : }
661 :
662 0 : static int mark_not_got(struct blk *blk, struct dpth *dpth)
663 : {
664 : const char *path;
665 :
666 0 : if(blk->got!=BLK_INCOMING) return 0;
667 0 : blk->got=BLK_NOT_GOT;
668 :
669 : // Need to get the data for this blk from the client.
670 : // Set up the details of where it will be saved.
671 0 : if(!(path=dpth_protocol2_mk(dpth))) return -1;
672 :
673 : // FIX THIS: make dpth give us the path in a uint8 array.
674 0 : blk->savepath=savepathstr_with_sig_to_uint64(path);
675 0 : blk->got_save_path=1;
676 0 : if(dpth_protocol2_incr_sig(dpth)) return -1;
677 0 : return 0;
678 : }
679 :
680 0 : static int mark_up_to_index(struct blist *blist,
681 : uint64_t index, struct dpth *dpth)
682 : {
683 : struct blk *blk;
684 :
685 : // Mark everything that was not got, up to the given index.
686 0 : for(blk=blist->blk_from_champ_chooser;
687 0 : blk && blk->index!=index; blk=blk->next)
688 0 : if(mark_not_got(blk, dpth))
689 0 : return -1;
690 0 : if(!blk)
691 : {
692 0 : logp("Could not find index from champ chooser: %lu\n", index);
693 0 : return -1;
694 : }
695 : //logp("Found index from champ chooser: %lu\n", index);
696 : //printf("index from cc: %d\n", index);
697 0 : blist->blk_from_champ_chooser=blk;
698 0 : return 0;
699 : }
700 :
701 0 : static int deal_with_sig_from_chfd(struct iobuf *rbuf, struct blist *blist,
702 : struct dpth *dpth)
703 : {
704 : static struct blk b;
705 0 : if(blk_set_from_iobuf_index_and_savepath(&b, rbuf))
706 0 : return -1;
707 :
708 0 : if(mark_up_to_index(blist, b.index, dpth))
709 0 : return -1;
710 0 : blist->blk_from_champ_chooser->savepath=b.savepath;
711 0 : blist->blk_from_champ_chooser->got=BLK_GOT;
712 0 : blist->blk_from_champ_chooser->got_save_path=1;
713 0 : return 0;
714 : }
715 :
716 0 : static int deal_with_wrap_up_from_chfd(struct iobuf *rbuf, struct blist *blist,
717 : struct dpth *dpth)
718 : {
719 : static struct blk b;
720 0 : if(blk_set_from_iobuf_wrap_up(&b, rbuf))
721 0 : return -1;
722 :
723 0 : if(mark_up_to_index(blist, b.index, dpth)) return -1;
724 0 : if(mark_not_got(blist->blk_from_champ_chooser, dpth)) return -1;
725 0 : return 0;
726 : }
727 :
728 0 : static int deal_with_read_from_chfd(struct asfd *chfd,
729 : struct blist *blist, uint64_t *wrap_up, struct dpth *dpth,
730 : struct cntr *cntr)
731 : {
732 0 : int ret=-1;
733 :
734 : // Deal with champ chooser read here.
735 : //printf("read from cc: %s\n", chfd->rbuf->buf);
736 0 : switch(chfd->rbuf->cmd)
737 : {
738 : case CMD_SIG:
739 : // Get these for blks that the champ chooser has found.
740 0 : if(deal_with_sig_from_chfd(chfd->rbuf, blist, dpth))
741 0 : goto end;
742 0 : cntr_add_same(cntr, CMD_DATA);
743 0 : break;
744 : case CMD_WRAP_UP:
745 0 : if(deal_with_wrap_up_from_chfd(chfd->rbuf, blist, dpth))
746 0 : goto end;
747 0 : break;
748 : default:
749 0 : iobuf_log_unexpected(chfd->rbuf, __func__);
750 0 : goto end;
751 : }
752 0 : ret=0;
753 : end:
754 0 : iobuf_free_content(chfd->rbuf);
755 0 : return ret;
756 : }
757 :
758 0 : static struct asfd *get_asfd_from_list_by_fdtype(struct async *as,
759 : enum asfd_fdtype fdtype)
760 : {
761 : struct asfd *a;
762 0 : for(a=as->asfd; a; a=a->next)
763 0 : if(a->fdtype==fdtype) return a;
764 0 : return NULL;
765 : }
766 :
767 0 : int backup_phase2_server_protocol2(struct async *as, struct sdirs *sdirs,
768 : int resume, struct conf **confs)
769 : {
770 0 : int ret=-1;
771 0 : uint8_t end_flags=0;
772 0 : struct slist *slist=NULL;
773 : struct iobuf wbuf;
774 0 : struct dpth *dpth=NULL;
775 0 : man_off_t *p1pos=NULL;
776 0 : struct manios *manios=NULL;
777 : // This is used to tell the client that a number of consecutive blocks
778 : // have been found and can be freed.
779 0 : uint64_t wrap_up=0;
780 0 : struct asfd *asfd=as->asfd;
781 : struct asfd *chfd;
782 0 : struct cntr *cntr=get_cntr(confs);
783 0 : if(get_int(confs[OPT_BREAKPOINT])>=2000
784 0 : && get_int(confs[OPT_BREAKPOINT])<3000)
785 : {
786 0 : breaking=get_int(confs[OPT_BREAKPOINT]);
787 0 : breakcount=breaking-2000;
788 : }
789 :
790 0 : chfd=get_asfd_from_list_by_fdtype(as, ASFD_FD_SERVER_TO_CHAMP_CHOOSER);
791 :
792 0 : logp("Phase 2 begin (recv backup data)\n");
793 :
794 0 : if(!(dpth=dpth_alloc())
795 0 : || dpth_protocol2_init(dpth,
796 0 : sdirs->data, get_int(confs[OPT_MAX_STORAGE_SUBDIRS])))
797 0 : goto end;
798 0 : if(resume && !(p1pos=do_resume(sdirs, dpth, confs)))
799 0 : goto end;
800 :
801 0 : if(!(manios=manios_open_phase2(sdirs, p1pos, PROTO_2))
802 0 : || !(slist=slist_alloc()))
803 0 : goto end;
804 :
805 0 : iobuf_free_content(asfd->rbuf);
806 :
807 0 : memset(&wbuf, 0, sizeof(struct iobuf));
808 0 : while(!(end_flags&END_BACKUP))
809 : {
810 0 : if(maybe_add_from_scan(manios, slist, chfd))
811 0 : goto end;
812 :
813 0 : if(!wbuf.len)
814 : {
815 0 : if(get_wbuf_from_sigs(&wbuf, slist, &end_flags))
816 0 : goto end;
817 0 : if(!wbuf.len)
818 : {
819 : get_wbuf_from_files(&wbuf, slist,
820 0 : manios, &end_flags);
821 : }
822 : }
823 :
824 0 : if(wbuf.len
825 0 : && asfd->append_all_to_write_buffer(asfd, &wbuf)==APPEND_ERROR)
826 0 : goto end;
827 :
828 0 : if(append_for_champ_chooser(chfd, slist->blist, end_flags))
829 0 : goto end;
830 :
831 0 : if(as->read_write(as))
832 : {
833 0 : logp("error in %s\n", __func__);
834 0 : goto end;
835 : }
836 :
837 0 : while(asfd->rbuf->buf)
838 : {
839 0 : if(deal_with_read(asfd->rbuf, slist, cntr,
840 0 : &end_flags, dpth)) goto end;
841 : // Get as much out of the readbuf as possible.
842 0 : if(asfd->parse_readbuf(asfd)) goto end;
843 : }
844 0 : while(chfd->rbuf->buf)
845 : {
846 0 : if(deal_with_read_from_chfd(chfd,
847 0 : slist->blist, &wrap_up, dpth, cntr)) goto end;
848 : // Get as much out of the readbuf as possible.
849 0 : if(chfd->parse_readbuf(chfd)) goto end;
850 : }
851 :
852 0 : if(write_to_changed_file(asfd, chfd, manios,
853 0 : slist, end_flags))
854 0 : goto end;
855 : }
856 :
857 : // Hack: If there are some entries left after the last entry that
858 : // contains block data, it will not be written to the changed file
859 : // yet because the last entry of block data has not had
860 : // sb->protocol2->bend set.
861 0 : if(slist->head && slist->head->next)
862 : {
863 0 : slist->head=slist->head->next;
864 0 : if(write_to_changed_file(asfd, chfd, manios,
865 0 : slist, end_flags))
866 0 : goto end;
867 : }
868 :
869 0 : if(manios_close(&manios))
870 0 : goto end;
871 :
872 0 : if(slist->blist->head)
873 : {
874 : logp("ERROR: finishing but still want block: %lu\n",
875 0 : slist->blist->head->index);
876 0 : goto end;
877 : }
878 :
879 : // Need to release the last left. There should be one at most.
880 0 : if(dpth->head && dpth->head->next)
881 : {
882 : logp("ERROR: More data locks remaining after: %s\n",
883 0 : dpth->head->save_path);
884 0 : goto end;
885 : }
886 0 : if(dpth_release_all(dpth)) goto end;
887 :
888 0 : ret=0;
889 : end:
890 0 : logp("End backup\n");
891 0 : slist_free(&slist);
892 0 : iobuf_free_content(asfd->rbuf);
893 0 : iobuf_free_content(chfd->rbuf);
894 0 : dpth_free(&dpth);
895 0 : manios_close(&manios);
896 0 : man_off_t_free(&p1pos);
897 0 : return ret;
898 : }
|
