gget

gget Commit Details

Date:2016-08-12 18:10:03 (3 years 11 months ago)
Author:Grégory Soutadé
Branch:master
Commit:c3c7945ddf044ecafd4eb2bbd8f90b7ff87d18ff
Parents: a5d01d4ddf02544ceaacda6967f6be9dae384c77
Message:Add -m switch Disaply size of downloaded file in an human way

Changes:
Mgget.c (17 diffs)

File differences

gget.c
11
2
2
33
44
55
......
8686
8787
8888
89
8990
9091
9192
......
101102
102103
103104
105
104106
105107
106108
......
216218
217219
218220
219
220
221
222
221223
222224
223225
......
226228
227229
228230
229
230
231
232
231
232
233
233234
234235
235236
236
237237
238238
239239
......
248248
249249
250250
251
252
253
254
255
256
251
252
253
254
255
256
257257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
258282
259283
260284
......
262286
263287
264288
265
289
266290
267291
268292
......
284308
285309
286310
311
287312
288313
289314
......
371396
372397
373398
399
374400
375401
376402
......
452478
453479
454480
455
481
456482
457483
458484
459485
460486
461487
488
462489
463490
464491
......
467494
468495
469496
470
497
471498
472499
473500
474501
475
476
477
502
503
504
505
478506
479507
480508
......
484512
485513
486514
487
515
488516
489517
490518
......
499527
500528
501529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
502563
503564
504565
......
539600
540601
541602
542
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
543626
544627
545628
......
561644
562645
563646
564
647
648
565649
566650
567651
652
653
654
655
656
568657
569658
570659
......
583672
584673
585674
586
675
676
587677
588678
589679
......
665755
666756
667757
668
758
759
669760
670761
671762
/*
Copyright 2014 Grégory Soutadé
Copyright 2014-2016 Grégory Soutadé
This file is part of gget.
typedef struct {
curl_off_t dltotal;
curl_off_t dlnow;
curl_off_t dllast;
unsigned speed;
} stats_t ;
unsigned already_downloaded;
unsigned start;
unsigned end;
unsigned max_chunk_size;
curl_off_t max_speed;
stats_t* stats;
} transfert_t;
curl_easy_getinfo(t->curl, CURLINFO_SPEED_DOWNLOAD, &speed_d);
t->stats[t->id].dltotal = dltotal + t->already_downloaded;
t->stats[t->id].dlnow = dlnow + t->already_downloaded;
t->stats[t->id].dlnow += (dlnow - t->stats[t->id].dllast);
t->stats[t->id].dllast = dlnow;
t->stats[t->id].speed = speed_d;
return 0;
void* do_transfert(transfert_t* t)
{
CURLcode res;
char range[50];
snprintf(range, sizeof(range), "%u-%u", t->start, t->end);
char range[64];
unsigned start, end, chunk_size;
curl_easy_setopt(t->curl, CURLOPT_USERAGENT, t->user_agent);
curl_easy_setopt(t->curl, CURLOPT_URL, t->url);
curl_easy_setopt(t->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(t->curl, CURLOPT_RANGE, range);
if (t->max_speed)
curl_easy_setopt(t->curl, CURLOPT_MAX_RECV_SPEED_LARGE, t->max_speed);
curl_easy_setopt(t->curl, CURLOPT_XFERINFOFUNCTION, progress_cb);
curl_easy_setopt(t->curl, CURLOPT_XFERINFODATA, t);
/* Perform the request, res will get the return code */
res = curl_easy_perform(t->curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
start = t->start;
if (t->max_chunk_size && (t->end - t->start) > t->max_chunk_size)
chunk_size = t->max_chunk_size;
else
chunk_size = t->end - t->start;
end = start + chunk_size;
while (start < t->end)
{
snprintf(range, sizeof(range), "%u-%u", start, end);
curl_easy_setopt(t->curl, CURLOPT_RANGE, range);
/* Perform the request, res will get the return code */
res = curl_easy_perform(t->curl);
/* Check for errors */
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
break;
}
start += chunk_size + 1;
if (t->max_chunk_size && (t->end - start) > t->max_chunk_size)
chunk_size = t->max_chunk_size;
else
chunk_size = t->end - start;
end = start + chunk_size;
t->stats[t->id].dllast = 0;
}
return NULL;
}
char* url, char* filename,
unsigned start, unsigned end,
unsigned max_speed, char* user_agent,
int* exists)
unsigned max_chunk_size, int* exists)
{
// filename + . + number + \0
unsigned filename_size = strlen(filename)+1+3+1;
t->end = end;
t->max_speed = max_speed;
t->user_agent = user_agent;
t->max_chunk_size = max_chunk_size;
if (stat(t->tmp_filename, &s))
{
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
if (!quiet)
curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
static void usage(char* program_name)
{
printf("%s: Parallel HTTP file download\n", program_name);
printf("usage: %s [-n nb_threads] [-l speed_limit] [-o out_filename] [-u user_agent] [-q] [-h] url\n",
printf("usage: %s [-n nb_threads] [-l speed_limit] [-o out_filename] [-u user_agent] [-m max_chunk_size[kKmMgG] ] [-q] [-h] url\n",
program_name);
printf("\t-n : Specify number of threads (default : %d)\n", DEFAULT_NB_THREADS);
printf("\t-l : Download speed limit for all threads (not per thread)\n");
printf("\t-o : Out filename, default is retrieved by GET request or '%s' if not found\n",
DEFAULT_OUT_FILENAME);
printf("\t-u : User agent, default is '%s'\n", DEFAULT_USER_AGENT);
printf("\t-m : Max chunk size in bytes\n");
printf("\t-q : Quiet mode\n");
printf("\t-h : Display help\n");
}
{
pthread_t display_thread = 0;
unsigned nb_threads = DEFAULT_NB_THREADS;
char* user_agent = strdup(DEFAULT_USER_AGENT);
char* user_agent = strdup(DEFAULT_USER_AGENT), *endptr;
stats_t* stats = NULL;
int ret = -1, i;
stats_params_t stats_params;
transfert_t* transferts = NULL;
unsigned total_size, thread_size;
unsigned start, end, max_speed = 0, quiet = 0;
char* out_filename = NULL, *url = NULL;
unsigned total_size, thread_size, multiplier=1;
double displayed_size;
unsigned start, end, max_speed = 0, quiet = 0, max_chunk_size = 0;
char* out_filename = NULL, *url = NULL, c;
char* suffix = "B";
void* res;
int opt;
int continuous_mode=0;
int exists;
while ((opt = getopt(argc, argv, "hn:l:o:u:q")) != -1) {
while ((opt = getopt(argc, argv, "hn:l:o:u:qm:")) != -1) {
switch (opt) {
case 'n':
nb_threads = atoi(optarg);
case 'l':
max_speed = atoi(optarg);
break;
case 'm':
if (strlen(optarg) > 1)
{
c = optarg[strlen(optarg)-1];
if (c < '0' || c > '9')
{
switch(c)
{
case 'g':
case 'G':
multiplier *= 1024;
case 'm':
case 'M':
multiplier *= 1024;
case 'k':
case 'K':
multiplier *= 1024;
optarg[strlen(optarg)-1] = 0;
break;
default:
usage(argv[0]);
return 1;
}
}
}
max_chunk_size = strtoul(optarg, &endptr, 0);
if (*endptr)
{
usage(argv[0]);
return 1;
}
max_chunk_size *= multiplier;
break;
case 'o':
out_filename = strdup(optarg);
break;
goto end;
}
else
printf("Save in '%s'\n\n", out_filename);
{
displayed_size = (double)total_size;
suffix = "B";
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "kB";
}
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "MB";
}
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "GB";
}
printf("Save in '%s' (%.2f%s)\n\n", out_filename, displayed_size, suffix);
}
stats = malloc(sizeof(*stats)*nb_threads);
memset(stats, 0, sizeof(*stats)*nb_threads);
end = total_size;
ret = configure_transfert(i, &transferts[i], url, out_filename,
start, end, max_speed, user_agent, &exists);
start, end, max_speed, user_agent,
max_chunk_size, &exists);
if (ret)
goto end;
transferts[i].stats[i].dltotal = transferts[i].already_downloaded +
(end - start);
transferts[i].stats[i].dlnow = transferts[i].already_downloaded;
transferts[i].stats[i].dllast = 0;
// First set continuous mode
if (i == 0)
{
// Check for last temporary file
configure_transfert(i, &transferts[i], url, out_filename,
start, end, max_speed, user_agent, &exists);
start, end, max_speed, user_agent, max_chunk_size,
&exists);
unlink(transferts[i].tmp_filename);
if (exists)
if (transferts)
free(transferts);
printf("\n");
if (!quiet)
printf("\n");
return ret;
}

Archive Download the corresponding diff file

Branches

Tags