gget

gget Git Source Tree

Root/gget.c

1/*
2 Copyright 2014-2016 Grégory Soutadé
3
4 This file is part of gget.
5
6 gget is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 gget is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with gget. If not, see <http://www.gnu.org/licenses/>.
18*/
19
20#include <sys/types.h>
21#include <sys/stat.h>
22#include <sys/time.h>
23#include <fcntl.h>
24#include <stdio.h>
25#include <unistd.h>
26#include <stdlib.h>
27#include <string.h>
28#include <curl/curl.h>
29#include <libgen.h>
30
31#define DEFAULT_USER_AGENT "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0"
32#define DEFAULT_NB_THREADS 3
33#define MAX_NB_THREADS 10
34#define DEFAULT_OUT_FILENAME "gget.out"
35
36#ifdef WIN32
37#include <windows.h>
38#define GNU_ERR ""
39
40typedef HANDLE pthread_t;
41typedef void pthread_attr_t;
42
43static int get_console_width()
44{
45 RECT r;
46 HWND console = GetConsoleWindow();
47
48 GetWindowRect(console, &r);
49
50 return r.right-r.left;
51}
52
53static int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
54 void *(*start_routine) (void *), void *arg)
55{
56 *thread = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)start_routine,
57 arg, 0, NULL);
58
59 return 0;
60}
61
62static int pthread_join(pthread_t thread, void **retval)
63{
64 WaitForSingleObject(thread, 0);
65 GetExitCodeThread(thread,(LPDWORD)retval);
66 CloseHandle(thread);
67
68 return 0;
69}
70
71#else
72#include <sys/ioctl.h>
73#include <pthread.h>
74
75#define GNU_ERR " (%m)"
76static int get_console_width()
77{
78 struct winsize w;
79
80 ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
81
82 return w.ws_col;
83}
84#endif
85
86typedef struct {
87 curl_off_t dltotal;
88 curl_off_t dlnow;
89 curl_off_t dllast;
90 unsigned speed;
91} stats_t ;
92
93typedef struct {
94 pthread_t thread;
95 char* tmp_filename;
96 //
97 CURL *curl;
98 char* url;
99 char* user_agent;
100 int fd;
101 int id;
102 unsigned already_downloaded;
103 unsigned start;
104 unsigned end;
105 unsigned max_chunk_size;
106 curl_off_t max_speed;
107 stats_t* stats;
108} transfert_t;
109
110typedef struct {
111 unsigned nb_threads;
112 stats_t* stats;
113 unsigned end;
114} stats_params_t;
115
116static void* display_stats(stats_params_t* p)
117{
118 unsigned speed, percent, time_left, max_time_left;
119 unsigned total_percent, hours, minutes, seconds;
120 char* suffix;
121 stats_t* cur;
122 int i, nb_chars_printed;
123
124 while (!p->end)
125 {
126 // If the window has been resized
127 max_time_left = 0;
128 suffix = "B";
129 total_percent = 0;
130 nb_chars_printed = 0;
131
132 printf("\r");
133 for(i=0; i<p->nb_threads; i++)
134 {
135 cur = &p->stats[i];
136
137 speed = cur->speed;
138
139 if (cur->dltotal)
140 percent = (cur->dlnow*100)/cur->dltotal;
141 else
142 percent = 0;
143
144 if (speed)
145 time_left = (cur->dltotal-cur->dlnow)/speed;
146 else
147 time_left = 0;
148
149 if (speed < (1024*1024))
150 {
151 speed /= 1024;
152 suffix = "kB";
153 }
154 else
155 {
156 speed /= 1024*1024;
157 suffix = "MB";
158 }
159
160 nb_chars_printed +=
161 printf("T%d: %02u%% %u%s/s ",
162 i, percent, speed, suffix);
163
164 if (time_left > max_time_left)
165 max_time_left = time_left;
166
167 total_percent += percent/p->nb_threads;
168 }
169
170 nb_chars_printed += printf(" Total: %u%% ", total_percent);
171
172 if (max_time_left < 60)
173 {
174 nb_chars_printed += printf("eta %us", max_time_left);
175 }
176 else
177 {
178 seconds = max_time_left % 60;
179 max_time_left /= 60;
180
181 if (max_time_left < 60)
182 {
183 minutes = max_time_left;
184 nb_chars_printed += printf("eta %um %us", minutes, seconds);
185 }
186 else
187 {
188 minutes = max_time_left % 60;
189 hours = max_time_left / 60;
190 nb_chars_printed += printf("eta %uh %um %us", hours, minutes, seconds);
191 }
192 }
193
194 for(i=nb_chars_printed; i<get_console_width(); i++)
195 printf(" ");
196
197 fflush(stdout);
198
199 sleep(1);
200 }
201
202 return NULL;
203}
204
205static size_t write_cb(void *contents, size_t size, size_t nmemb, void *userp)
206{
207 int ret;
208 ret = write(((transfert_t*)userp)->fd, contents, size*nmemb);
209 if (ret < 0)
210 printf("Error write" GNU_ERR "\n");
211 return ret;
212}
213
214static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
215{
216 transfert_t* t = (transfert_t*)clientp;
217 double speed_d;
218
219 curl_easy_getinfo(t->curl, CURLINFO_SPEED_DOWNLOAD, &speed_d);
220
221 t->stats[t->id].dlnow += (dlnow - t->stats[t->id].dllast);
222 t->stats[t->id].dllast = dlnow;
223 t->stats[t->id].speed = speed_d;
224
225 return 0;
226}
227
228void* do_transfert(transfert_t* t)
229{
230 CURLcode res;
231 char range[64];
232 unsigned start, end, chunk_size;
233
234 curl_easy_setopt(t->curl, CURLOPT_USERAGENT, t->user_agent);
235 curl_easy_setopt(t->curl, CURLOPT_URL, t->url);
236 curl_easy_setopt(t->curl, CURLOPT_FOLLOWLOCATION, 1L);
237
238 if (t->max_speed)
239 curl_easy_setopt(t->curl, CURLOPT_MAX_RECV_SPEED_LARGE, t->max_speed);
240
241 /* send all data to this function */
242 curl_easy_setopt(t->curl, CURLOPT_WRITEFUNCTION, write_cb);
243
244 /* we pass our 'chunk' struct to the callback function */
245 curl_easy_setopt(t->curl, CURLOPT_WRITEDATA, (void*)t);
246
247 curl_easy_setopt(t->curl, CURLOPT_NOPROGRESS, 0L);
248 curl_easy_setopt(t->curl, CURLOPT_XFERINFOFUNCTION, progress_cb);
249 curl_easy_setopt(t->curl, CURLOPT_XFERINFODATA, t);
250
251 start = t->start;
252 if (t->max_chunk_size && (t->end - t->start) > t->max_chunk_size)
253 chunk_size = t->max_chunk_size;
254 else
255 chunk_size = t->end - t->start;
256 end = start + chunk_size;
257
258 while (start < t->end)
259 {
260 snprintf(range, sizeof(range), "%u-%u", start, end);
261 curl_easy_setopt(t->curl, CURLOPT_RANGE, range);
262
263 /* Perform the request, res will get the return code */
264 res = curl_easy_perform(t->curl);
265 /* Check for errors */
266 if(res != CURLE_OK)
267 {
268 fprintf(stderr, "curl_easy_perform() failed: %s\n",
269 curl_easy_strerror(res));
270 break;
271 }
272
273 start += chunk_size + 1;
274 if (t->max_chunk_size && (t->end - start) > t->max_chunk_size)
275 chunk_size = t->max_chunk_size;
276 else
277 chunk_size = t->end - start;
278 end = start + chunk_size;
279 t->stats[t->id].dllast = 0;
280 }
281
282 return NULL;
283}
284
285static int configure_transfert(int id, transfert_t* t,
286 char* url, char* filename,
287 unsigned start, unsigned end,
288 unsigned max_speed, char* user_agent,
289 unsigned max_chunk_size, int* exists)
290{
291 // filename + . + number + \0
292 unsigned filename_size = strlen(filename)+1+3+1;
293 struct stat s;
294
295 t->curl = curl_easy_init();
296
297 if(!t->curl)
298 return -1;
299
300 t->url = url;
301
302 t->tmp_filename = malloc(filename_size);
303 snprintf(t->tmp_filename, filename_size, "%s.%d",
304 filename, id);
305 t->id = id;
306
307 t->start = start;
308 t->end = end;
309 t->max_speed = max_speed;
310 t->user_agent = user_agent;
311 t->max_chunk_size = max_chunk_size;
312
313 if (stat(t->tmp_filename, &s))
314 {
315 *exists = 0;
316
317 t->fd = open(t->tmp_filename,
318 O_WRONLY | O_CREAT,
319 S_IRUSR|S_IWUSR);
320 t->already_downloaded = 0;
321 }
322 else
323 {
324 *exists = 1;
325
326 t->fd = open(t->tmp_filename,
327 O_WRONLY | O_APPEND);
328
329 t->start += s.st_size;
330 t->already_downloaded = s.st_size;
331 }
332
333 if (!t->fd)
334 {
335 printf("Opening '%s' failed" GNU_ERR "\n",
336 t->tmp_filename);
337 return -1;
338 }
339
340 return 0;
341}
342
343static int free_transfert(transfert_t* t)
344{
345 if (t)
346 {
347 if (t->curl)
348 curl_easy_cleanup(t->curl);
349
350 if (t->tmp_filename)
351 free(t->tmp_filename);
352 }
353
354 return 0;
355}
356
357static void merge_files(transfert_t* transferts, int nb_threads,
358 char* out_filename)
359{
360 int fd, fd_tmp, i;
361 struct stat s;
362 char* buffer;
363
364 rename(transferts[0].tmp_filename, out_filename);
365 fd = open(out_filename, O_WRONLY | O_APPEND);
366
367 for(i=1; i<nb_threads; i++)
368 {
369 fd_tmp = open(transferts[i].tmp_filename, O_RDONLY);
370 stat(transferts[i].tmp_filename, &s);
371 buffer = malloc(s.st_size);
372 read(fd_tmp, buffer, s.st_size);
373 write(fd, buffer, s.st_size);
374 free(buffer);
375 close(fd_tmp);
376 unlink(transferts[i].tmp_filename);
377 }
378 close(fd);
379}
380
381static int get_file_info(char* url, char* user_agent,
382 char** filename, unsigned* size,
383 unsigned quiet)
384{
385 double length;
386 CURLcode res;
387 int ret = 0;
388 unsigned response_code;
389 char* real_name = NULL;
390 CURL* curl = curl_easy_init();
391
392 if(!curl)
393 return -1;
394
395 curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
396 curl_easy_setopt(curl, CURLOPT_URL, url);
397 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
398 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
399 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
400 if (!quiet)
401 curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
402
403 res = curl_easy_perform(curl);
404
405 /* Check for errors */
406 if(res != CURLE_OK)
407 {
408 fprintf(stderr, "curl_easy_perform() failed: %s\n",
409 curl_easy_strerror(res));
410 ret = -1;
411 goto end;
412 }
413
414 curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &response_code);
415
416 if (response_code != 200)
417 {
418 ret = -1;
419 goto end;
420 }
421
422 curl_easy_getinfo( curl,
423 CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length );
424 *size = length;
425
426 if (*filename == NULL)
427 {
428 curl_easy_getinfo( curl, CURLINFO_EFFECTIVE_URL, &real_name );
429
430 if (real_name)
431 {
432 *filename = strdup(basename(real_name));
433
434 if (!strcmp(*filename, "."))
435 {
436 free(*filename);
437 *filename = strdup(DEFAULT_OUT_FILENAME);
438 printf("Filename not found, output to %s\n", DEFAULT_OUT_FILENAME);
439 }
440 }
441 }
442
443end:
444 curl_easy_cleanup(curl);
445
446 return ret;
447}
448
449static void find_free_file(char** filename)
450{
451 struct stat s;
452 int cur_index;
453 char* new_filename;
454 unsigned new_size;
455
456 if (stat(*filename, &s)) return ;
457
458 new_size = strlen(*filename) + 1 + 2 + 1;
459 new_filename = malloc(new_size);
460
461 for (cur_index = 1; cur_index < 100; cur_index++)
462 {
463 snprintf(new_filename, new_size, "%s.%d",
464 *filename, cur_index);
465
466 if (stat(new_filename, &s))
467 {
468 free(*filename);
469 *filename = new_filename;
470 return ;
471 }
472 }
473
474 free(*filename);
475 *filename = NULL;
476}
477
478static void usage(char* program_name)
479{
480 printf("%s: Parallel HTTP file download\n", program_name);
481 printf("usage: %s [-n nb_threads] [-l speed_limit] [-o out_filename] [-u user_agent] [-m max_chunk_size[kKmMgG]] [-q] [-h] url\n",
482 program_name);
483 printf("\t-n : Specify number of threads (default : %d)\n", DEFAULT_NB_THREADS);
484 printf("\t-l : Download speed limit for all threads (not per thread)\n");
485 printf("\t-o : Out filename, default is retrieved by GET request or '%s' if not found\n",
486 DEFAULT_OUT_FILENAME);
487 printf("\t-u : User agent, default is '%s'\n", DEFAULT_USER_AGENT);
488 printf("\t-m : Max chunk size in bytes\n");
489 printf("\t-q : Quiet mode\n");
490 printf("\t-h : Display help\n");
491}
492
493int main(int argc, char** argv)
494{
495 pthread_t display_thread = 0;
496 unsigned nb_threads = DEFAULT_NB_THREADS;
497 char* user_agent = strdup(DEFAULT_USER_AGENT), *endptr;
498 stats_t* stats = NULL;
499 int ret = -1, i;
500 stats_params_t stats_params;
501 transfert_t* transferts = NULL;
502 unsigned total_size, thread_size, multiplier=1;
503 double displayed_size;
504 unsigned start, end, max_speed = 0, quiet = 0, max_chunk_size = 0;
505 char* out_filename = NULL, *url = NULL, c;
506 char* suffix = "B";
507 void* res;
508 int opt;
509 struct timeval time_start, time_end;
510 struct tm * full_time;
511 double average_speed;
512 int continuous_mode=0;
513 int exists;
514
515 while ((opt = getopt(argc, argv, "hl:m:n:o:qu:")) != -1) {
516 switch (opt) {
517 case 'l':
518 max_speed = strtoul(optarg, &endptr, 0);
519 if (*endptr)
520 {
521 usage(argv[0]);
522 return 1;
523 }
524 break;
525 case 'm':
526 if (strlen(optarg) > 1)
527 {
528 c = optarg[strlen(optarg)-1];
529 if (c < '0' || c > '9')
530 {
531 switch(c)
532 {
533 case 'g':
534 case 'G':
535 multiplier *= 1024;
536 case 'm':
537 case 'M':
538 multiplier *= 1024;
539 case 'k':
540 case 'K':
541 multiplier *= 1024;
542 optarg[strlen(optarg)-1] = 0;
543 break;
544 default:
545 usage(argv[0]);
546 return 1;
547 }
548 }
549 }
550 max_chunk_size = strtoul(optarg, &endptr, 0);
551 if (*endptr)
552 {
553 usage(argv[0]);
554 return 1;
555 }
556 max_chunk_size *= multiplier;
557 break;
558 case 'n':
559 nb_threads = strtoul(optarg, &endptr, 0);
560 if (*endptr)
561 {
562 usage(argv[0]);
563 return 1;
564 }
565 if (nb_threads == 0)
566 nb_threads = DEFAULT_NB_THREADS;
567 else if (nb_threads > MAX_NB_THREADS)
568 {
569 printf("Max numberb of threads is %d\n", MAX_NB_THREADS);
570 nb_threads = MAX_NB_THREADS;
571 }
572 break;
573 case 'o':
574 out_filename = strdup(optarg);
575 break;
576 case 'q':
577 quiet = 1;
578 break;
579 case 'u':
580 user_agent = strdup(optarg);
581 break;
582 default:
583 case 'h':
584 usage(argv[0]);
585 return 0;
586 }
587 }
588
589 if (optind != argc-1)
590 {
591 usage(argv[0]);
592 return 0;
593 }
594 else
595 url = argv[optind];
596
597 if (get_file_info(url, user_agent,
598 &out_filename, &total_size,
599 quiet))
600 return -1;
601
602 if (!out_filename)
603 out_filename = strdup(DEFAULT_OUT_FILENAME);
604
605 find_free_file(&out_filename);
606
607 if (out_filename == NULL)
608 {
609 printf("Unable to find free file to write to !\n");
610 goto end;
611 }
612 else
613 {
614 displayed_size = (double)total_size;
615 suffix = "B";
616 if (displayed_size > (1024))
617 {
618 displayed_size /= 1024.0;
619 suffix = "kB";
620 }
621
622 if (displayed_size > (1024))
623 {
624 displayed_size /= 1024.0;
625 suffix = "MB";
626 }
627
628 if (displayed_size > (1024))
629 {
630 displayed_size /= 1024.0;
631 suffix = "GB";
632 }
633
634 printf("Save in '%s' (%.2f%s)\n\n", out_filename, displayed_size, suffix);
635 }
636
637 stats = malloc(sizeof(*stats)*nb_threads);
638 memset(stats, 0, sizeof(*stats)*nb_threads);
639
640 transferts = malloc(sizeof(*transferts)*(nb_threads+1));
641 memset(transferts, 0, sizeof(*transferts)*(nb_threads+1));
642
643 thread_size = total_size/nb_threads;
644 max_speed /= nb_threads;
645
646 for(i=0; i<nb_threads; i++)
647 {
648 transferts[i].stats = stats;
649
650 start = thread_size*i;
651 if (i < (nb_threads-1))
652 end = thread_size*(i+1)-1;
653 else
654 end = total_size;
655
656 ret = configure_transfert(i, &transferts[i], url, out_filename,
657 start, end, max_speed, user_agent,
658 max_chunk_size, &exists);
659 if (ret)
660 goto end;
661
662 transferts[i].stats[i].dltotal = transferts[i].already_downloaded +
663 (end - start);
664 transferts[i].stats[i].dlnow = transferts[i].already_downloaded;
665 transferts[i].stats[i].dllast = 0;
666
667 // First set continuous mode
668 if (i == 0)
669 {
670 continuous_mode = exists;
671 }
672 else
673 {
674 // Check valid continuous mode or not
675 if (exists ^ continuous_mode)
676 {
677 printf("Error : you already started to download this file with a different number of thread, please clear temporary files or restart with the same number of threads\n");
678 goto end;
679 }
680 }
681 }
682
683 // Check for last temporary file
684 configure_transfert(i, &transferts[i], url, out_filename,
685 start, end, max_speed, user_agent, max_chunk_size,
686 &exists);
687 unlink(transferts[i].tmp_filename);
688
689 if (exists)
690 {
691 printf("Error : you already started to download this file with a different number of thread, please clear temporary files or restart with the same number of threads\n");
692 free_transfert(&transferts[i]);
693 goto end;
694 }
695
696 stats_params.end = 0;
697 stats_params.nb_threads = nb_threads;
698 stats_params.stats = stats;
699
700 if (!quiet)
701 {
702 pthread_create(&display_thread, NULL,
703 (void*(*)(void*))display_stats,
704 (void*) &stats_params);
705 }
706
707 gettimeofday(&time_start, NULL);
708
709 for(i=0; i<nb_threads; i++)
710 {
711 pthread_create(&transferts[i].thread, NULL,
712 (void*(*)(void*))do_transfert,
713 (void*) &transferts[i]);
714 }
715
716 for(i=0; i<nb_threads; i++)
717 {
718 pthread_join(transferts[i].thread, &res);
719 }
720
721 gettimeofday(&time_end, NULL);
722
723 merge_files(transferts, nb_threads, out_filename);
724
725 if (!quiet)
726 {
727 full_time = localtime((const time_t*)&time_end.tv_sec);
728
729 average_speed = (double)total_size / (double)(time_end.tv_sec - time_start.tv_sec);
730
731 if (average_speed < (1024*1024))
732 {
733 average_speed /= 1024;
734 suffix = "kB";
735 }
736 else
737 {
738 average_speed /= 1024*1024;
739 suffix = "MB";
740 }
741
742 printf("\n\n%04d-%02d-%02d %02d:%02d:%02d (%.2f%s/s) '%s' saved\n",
743 full_time->tm_year+1900, full_time->tm_mon, full_time->tm_mday,
744 full_time->tm_hour, full_time->tm_min, full_time->tm_sec,
745 average_speed, suffix, out_filename);
746 }
747
748 ret = 0;
749
750end:
751 if (stats)
752 free(stats);
753
754 free(user_agent);
755 free(out_filename);
756
757 stats_params.end = 1;
758
759 for(i=0; i<nb_threads; i++)
760 free_transfert(&transferts[i]);
761
762 if (!quiet)
763 pthread_join(display_thread, &res);
764
765 if (transferts)
766 free(transferts);
767
768 if (!quiet)
769 printf("\n");
770
771 return ret;
772}

Archive Download this file

Branches

Tags