aget-devel/0040755000175000000000000000000010057031556012102 5ustar muratwheelaget-devel/AUTHORS0100755000175000000000000000045507614772646013177 0ustar muratwheel Initial Code by: Murat Balaban Current Maintainers: Murat Balaban Naveen N. Rao For questions and comments, and also bug reports, don't hesitate to mail murat@enderunix.org http://www.enderunix.org/ Fri Nov 22 07:43:40 EET 2002 aget-devel/Aget.c0100755000175000000000000003167010057022515013130 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Aget.h" #include "Misc.h" #include "Download.h" #include "Ftp.h" #include "Resume.h" #include "Data.h" #include "Signal.h" /*#define DEBUG */ extern struct thread_data *wthread; extern struct request *req; extern int fsuggested, nthreads; extern int bwritten; extern pthread_t hthread; extern char http_proxyhost[VALSIZE]; time_t t_start, t_finish; void startHTTP(struct request *req) { int i, ret, fd, diff_sec, nok = 0; long soffset, foffset; char *fmt; sigset_t set; sigfillset(&set); http_head_req(req); /* TODO:: We should create the helper thread here, but should take care * while resuming since wthread structure may not be allocated at all */ /* According to the content-length, get the * suggested number of threads to open. * if the user insists on his value, let it be that way, * use the user's value. */ ret = numofthreads(req->clength); if (fsuggested == 0) { if (ret == 0) nthreads = 1; else nthreads = ret; } wthread = (struct thread_data *)calloc(nthreads, sizeof(struct thread_data)); Log("Downloading %s (%d bytes) from site %s(%s:%d).\nNumber of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if ((fd = open(req->lfile, O_CREAT | O_RDWR | O_EXCL, S_IRWXU)) == -1) { if(errno == EEXIST) { char reply[MAXBUFSIZ]; again: fprintf(stderr, "File already exists! Overwrite?(y/n) "); scanf("%2s", reply); if(reply[0] == 'n') exit(1); else if(reply[0] == 'y') { if ((fd = open(req->lfile, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else goto again; } } if ((lseek(fd, req->clength - 1, SEEK_SET)) == -1) { fprintf(stderr, "get: couldn't lseek: %s\n", strerror(errno)); exit(1); } if ((write(fd, "0", 1)) == -1) { fprintf(stderr, "get: couldn't allocate space for download file: %s\n", strerror(errno)); exit(1); } /* Get the starting time, prepare GET format string, and start the threads */ fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); time(&t_start); retry: for (i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { soffset = calc_offset(req->clength, i, nthreads); foffset = calc_offset(req->clength, i + 1, nthreads); wthread[i].soffset = soffset; wthread[i].foffset = (i == nthreads - 1 ? req->clength : foffset); wthread[i].offset = wthread[i].soffset; wthread[i].sin.sin_family = AF_INET; wthread[i].sin.sin_addr.s_addr = inet_addr(req->ip); wthread[i].sin.sin_port = htons(req->port); wthread[i].fd = dup(fd); wthread[i].clength = req->clength; if(strlen(http_proxyhost) > 0) { char tmp[MAXBUFSIZ]; if(req->proto == PROTO_HTTP) { strcpy(tmp, "http://"); } else { strcpy(tmp, "ftp://"); } strncat(tmp, req->host, MAXBUFSIZ); strncat(tmp, req->url, MAXBUFSIZ); snprintf(fmt, GETREQSIZ, GETREQ, tmp, req->host, PROGVERSION, soffset); } else { snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, soffset); } strncpy(wthread[i].getstr, fmt, GETREQSIZ); #ifdef DEBUG printf("Start: %ld, Finish: %ld, Offset: %ld, Diff: %ld\n", wthread[i].soffset, wthread[i].foffset, wthread[i].offset, wthread[i].offset - wthread[i].soffset); #endif } /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); /* Create a thread for handling signals. * Should create the helper thread only after wthread structure is ready & sane. * Only then does saving the log file makes sense. */ if ((ret = pthread_create(&hthread, NULL, signal_waiter, NULL)) != 0) { fprintf(stderr, "startHTTP: cannot create signal_waiter thread: %s, exiting...\n", strerror(errno)); exit(-1); } for(i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { if(pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])) != 0) { Log("Thread creation error!"); exit(2); } pthread_testcancel(); /* Check if SIGINT caught & exit -- safe cancellation point */ } pthread_testcancel(); /* Wait for all of the threads to finish... */ for (i = 0; i < nthreads; i++) { if((int)wthread[i].tid != -1) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } /* Now set tid=-1, so that if an interrupt arrives midway through this loop, * signal handler won't wait for the same thread */ (int)wthread[i].tid = -1; pthread_testcancel(); } if (nok == nthreads) { /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); pthread_cancel(hthread); } else /* Restart threads that failed */ goto retry; free(fmt); /* Get the finish time, derive some stats */ time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%d Kb/sec)", diff_sec, (req->clength / diff_sec) / 1024); Log("Shutting down..."); close(fd); } void startFTP(struct request *req) { int i, ret, fd, diff_sec, nok = 0; long soffset, foffset; int head_sd; sigset_t set; sigfillset(&set); if(strlen(http_proxyhost) > 0) { startHTTP(req); } /* Should return the socket descriptor for the control connection in sd_c * and for the data connection in sd_d so that we can reuse it. That * reduces initial logon overhead. Currently returns a flag indicating * whether the server supports RESTart command. */ head_sd = -1; if((ret = ftp_head_req(req, &head_sd)) == 1) { /* REST command supported * According to the content-length, get the * suggested number of threads to open. * if the user insists on his value, let it be that way, * use the user's value. */ ret = numofthreads(req->clength); if (fsuggested == 0) { if (ret == 0) nthreads = 1; else nthreads = ret; } } else nthreads = 1; /* TODO:: We should create the helper thread here, but should take care * while resuming since wthread structure may not be allocated at all */ wthread = (struct thread_data *)calloc(nthreads, sizeof(struct thread_data)); Log("Downloading %s (%d bytes) from site %s(%s:%d).\nNumber of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if ((fd = open(req->lfile, O_CREAT | O_RDWR | O_EXCL, S_IRWXU)) == -1) { if(errno == EEXIST) { char reply[MAXBUFSIZ]; again: fprintf(stderr, "File already exists! Overwrite?(y/n) "); scanf("%2s", reply); if(reply[0] == 'n') exit(1); else if(reply[0] == 'y') { if ((fd = open(req->lfile, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else goto again; } } if ((lseek(fd, req->clength - 1, SEEK_SET)) == -1) { fprintf(stderr, "get: couldn't lseek: %s\n", strerror(errno)); exit(1); } if ((write(fd, "0", 1)) == -1) { fprintf(stderr, "get: couldn't allocate space for download file: %s\n", strerror(errno)); exit(1); } /* Get the starting time and start the threads */ time(&t_start); retry: for (i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { soffset = calc_offset(req->clength, i, nthreads); foffset = calc_offset(req->clength, i + 1, nthreads); wthread[i].head_sd = (i == 0 ? head_sd : -1); wthread[i].soffset = soffset; wthread[i].foffset = (i == nthreads - 1 ? req->clength : foffset); wthread[i].offset = wthread[i].soffset; wthread[i].sin.sin_family = AF_INET; wthread[i].sin.sin_addr.s_addr = inet_addr(req->ip); wthread[i].sin.sin_port = htons(req->port); wthread[i].fd = dup(fd); wthread[i].clength = req->clength; strncpy(wthread[i].username, req->username, MAXBUFSIZ-1); strncpy(wthread[i].password, req->password, MAXBUFSIZ-1); strncpy(wthread[i].url, req->url, MAXURLSIZ -1); #ifdef DEBUG printf("Start: %ld, Finish: %ld, Offset: %ld, Diff: %ld\n", wthread[i].soffset, wthread[i].foffset, wthread[i].offset, wthread[i].offset - wthread[i].soffset); #endif } /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); /* Create a thread for handling signals. * Should create the helper thread only after wthread structure is ready & sane. * Only then does saving the log file makes sense. */ if ((ret = pthread_create(&hthread, NULL, signal_waiter, NULL)) != 0) { fprintf(stderr, "main: cannot create signal_waiter thread: %s, exiting...\n", strerror(errno)); exit(-1); } for(i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { if(pthread_create(&(wthread[i].tid), NULL, ftp_get, &(wthread[i])) != 0) { Log("Thread creation error!"); exit(2); } pthread_testcancel(); /* Check if SIGINT caught & exit -- safe cancellation point */ } pthread_testcancel(); /* Wait for all of the threads to finish... */ for (i = 0; i < nthreads; i++) { if((int)wthread[i].tid != -1) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } /* Now set tid=-1, so that if an interrupt arrives midway through this loop, * signal handler won't wait for the same thread */ (int)wthread[i].tid = -1; pthread_testcancel(); } if (nok == nthreads) { /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); pthread_cancel(hthread); } else /* Restart threads that failed */ goto retry; /* Get the finish time, derive some stats */ time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%d Kb/sec)", diff_sec, (req->clength / diff_sec) / 1024); Log("Shutting down..."); close(fd); } /* Can be used for resuming any download */ void resumeDownload(struct hist_data *h, int proto) { int i, fd, diff_sec, nok = 0, ret; char *fmt; sigset_t set; sigfillset(&set); nthreads = h->nthreads; fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data)); memcpy(req, &h->req, sizeof(struct request)); memcpy(wthread, h->wthread, sizeof(struct thread_data) * nthreads); Log("Resuming download %s (%d bytes) from site %s(%s:%d).\nNumber of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if ((fd = open(req->lfile, O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } time(&t_start); retry: for (i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { wthread[i].head_sd = -1; wthread[i].soffset = wthread[i].offset; wthread[i].fd = dup(fd); snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, wthread[i].offset); strncpy(wthread[i].getstr, fmt, GETREQSIZ); #ifdef DEBUG printf("Start: %ld, Finish: %ld, Offset: %ld, Diff: %ld\n", wthread[i].soffset, wthread[i].foffset, wthread[i].offset, wthread[i].offset - wthread[i].soffset); #endif } /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); /* Create a thread for handling signals. * Should create the helper thread only after wthread structure is ready & sane. * Only then does saving the log file makes sense. */ if ((ret = pthread_create(&hthread, NULL, signal_waiter, NULL)) != 0) { fprintf(stderr, "main: cannot create signal_waiter thread: %s, exiting...\n", strerror(errno)); exit(-1); } for(i = 0; i < nthreads && wthread[i].status | STAT_OK; i++) { if(proto == PROTO_FTP) pthread_create(&(wthread[i].tid), NULL, ftp_get, &(wthread[i])); else if(proto == PROTO_HTTP) pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])); pthread_testcancel(); } pthread_testcancel(); for (i = 0; i < nthreads; i++) { if((int)wthread[i].tid != -1) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } /* Now set tid=-1, so that if an interrupt arrives midway through this loop, * signal handler won't wait for the same thread */ (int)wthread[i].tid = -1; pthread_testcancel(); } if (nok == nthreads) { /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &set, NULL); pthread_cancel(hthread); } else /* Restart threads that failed */ goto retry; time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%d Kb/sec)", diff_sec, ((req->clength - h->bwritten) / diff_sec) / 1024); Log("Total download time: %d seconds. Overall speed: %d Kb/sec", diff_sec+req->time_taken, (req->clength/(diff_sec+req->time_taken)) / 1024); Log("Shutting down..."); close(fd); } aget-devel/Aget.h0100755000175000000000000000062307603413754013143 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Aget.h,v 1.4 2002/12/28 21:30:20 murat Exp $ * */ #ifndef AGET_H #define AGET_H #include "Data.h" #include "Resume.h" void startHTTP(struct request *); void startFTP(struct request *); void resumeDownload(struct hist_data *, int ); #endif aget-devel/COPYING0100755000175000000000000000347007567343561013157 0ustar muratwheel Copyright (c) 2002 , Murat Balaban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the EnderUNIX Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Fri Nov 22 07:44:00 EET 2002 aget-devel/ChangeLog0100755000175000000000000000553307763217016013671 0ustar muratwheelEnderUNIX Aget ChangeLog - extern int errno's replaced with #include directives for newer Linux versions compatibility. - Patch by Patrick MARIE mycroft virgaria.org>, which initializes struct request pointer. - Patch by Gurer Ozen to avoid excessive CPU consuming while downloading from very slow sites. - HTTP Proxy support (without authorization) - progress bar is now in one line. - RC file support. Some configuration parameters can be set in the .agetrc file which will be placed on the user's home directory. - Ftp downloads utilize the first head connection - Naveen N. Rao becomes the second developer for aget. - Aget supports ftp downloads now. (Passive only). - Thread exit status evaluation. Automatic restarting of threads. - Heavy changes on Signal Handling - Log file is now created in the user's home directory. - Progress bar is actived via alarm(2) now. * Sat Nov 23 - MaxOSX support - get() and resume_get() functions are replaced with startHTTP(), startFTP, resumeHTTP() and resumeFTP() respectively. * Fri Nov 22 07:44:20 EET 2002 Version 0.4 adds/changes the following: - Fixed bugs in Resume code. - Fixed miscalculations in the progress indicator * Fri Sep 20 18:11:11 EEST 2002 Version 0.39 adds/changes the following: - Hafiye is not 100 % POSIX compliant now. pread/pwrite functions are now used to simultaneously write to the local file. - Because of pwrite, the local file is not assembled at the end now, it's assembled at the same time. This speeds up large downloads; since previously lots of I/O was used to assemble the file. This is now decreased by at least fifty percent. - Solaris bug fixed. - Resume support. - Some more options: -p [port number to connect to] -l [local file name, if different from the remote file] -f [force use of suggested number of threads] - A progress indicator has been added. - Can guess the number of threads needed depending on file size - Code has been re-arranged. Since aget starts to become larger, that was a need. - A README-Developer has been added for developers who want to contribute to aget. * Fri May 3 23:25:08 EEST 2002 Version 0.2p1 adds/changes the following: - getopt() problem is now solved, The program should really run on Posix systems. I've tested it on Linux, FreeBSD ans Solaris. - If no thread number is supplied, it is defaulted to 1. - A Solaris Makefile is added to the tarball. * Wed May 1 18:15:32 EEST 2002 Version 0.2 adds/changes the following: - File is now created seperately in each thread and later, it is assembled. Thus, some overhead from locking and unlocking of the mutex has been decreased. * Wed May 1 10:43:44 EEST 2002 First Public Release: 0.1 aget-devel/Data.h0100755000175000000000000000256107615201053013125 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Data.h,v 1.8 2003/01/27 09:55:55 murat Exp $ * */ #ifndef DATA_H #define DATA_H #include #include #include #define SERVER_TYPE_UNIX 1 #define SERVER_TYPE_NT 2 #include "Defs.h" typedef struct request { char host[MAXHOSTSIZ]; /* Remote host */ char url[MAXURLSIZ]; /* URL */ char file[MAXBUFSIZ]; /* file name */ char lfile[MAXBUFSIZ]; /* if local file name is specified */ char ip[MAXIPSIZ]; /* Remote IP */ char username[MAXBUFSIZ]; char password[MAXBUFSIZ]; int port; int ftp_server_type; int clength; /* Content-length */ unsigned char proto; /* Protocol */ time_t time_taken; /* Useful when job is resumed */ } request; typedef struct thread_data { struct sockaddr_in sin; char getstr[GETREQSIZ]; long soffset; /* Start offset */ long foffset; /* Finish offset */ long offset; /* Current Offset */ long clength; /* Content Length */ int fd; pthread_t tid; /* Thread ID */ unsigned char status; /* thread exit status */ char username[MAXBUFSIZ]; /* Used in ftp_get() */ char password[MAXBUFSIZ]; /* Used in ftp_get() */ char url[MAXURLSIZ]; /* Used in ftp_get() */ int head_sd; /* Used for reusing head connection */ } thread_data; #endif aget-devel/Defs.h0100755000175000000000000000164310056623007013135 0ustar muratwheel/* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Defs.h,v 1.12 2004/05/31 12:33:11 murat Exp $ * */ #ifndef DEFS_H #define DEFS_H enum { GETREQSIZ = 256, GETRECVSIZ = 8192, MAXBUFSIZ = 8192, HEADREQSIZ = 512, MAXURLSIZ = 1024, MAXHOSTSIZ = 1024, MAXIPSIZ = 16, MAXTHREADS = 10, VALSIZE = 256, HTTPPORT = 80, UNKNOWNREQ = 2, FTPREQ = 21, PROTO_HTTP = 0xFF, PROTO_FTP = 0x00, STAT_OK = 0xFF, /* Download completed successfully */ STAT_INT = 0x0F, /* ^C caught, download interrupted */ STAT_ERR = 0x00 /* Download finished with error */ }; #define PROGVERSION "EnderUNIX Aget 0.59" #define HEADREQ "HEAD %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nConnection: close\r\n\r\n" #define GETREQ "GET %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nRange: bytes=%ld-\r\nConnection: close\r\n\r\n" #endif aget-devel/Download.c0100755000175000000000000000730210056620061014011 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * */ #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" #include "Download.h" extern sigset_t signal_set; unsigned int bwritten = 0; pthread_mutex_t bwritten_mutex = PTHREAD_MUTEX_INITIALIZER; void *http_get(void *arg) { struct thread_data *td; int sd; char *rbuf, *s; int dr, dw, i; long foffset; pthread_t tid; sigset_t set; tid = pthread_self(); /* Block out all signals */ sigfillset(&set); pthread_sigmask(SIG_BLOCK, &set, NULL); /* Set Cancellation Type to Asynchronous */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); td = (struct thread_data *)arg; foffset = td->foffset; if(td->soffset < 0 || td->soffset >= td->foffset) { /* If download complete */ td->status = STAT_OK; /* Tell that download is OK. */ pthread_exit((void *)1); return NULL; } rbuf = (char *)calloc(MAXBUFSIZ, sizeof(char)); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log(" socket creation failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((connect(sd, (const struct sockaddr *)&td->sin, sizeof(struct sockaddr))) == -1) { Log(" connection failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((send(sd, td->getstr, strlen(td->getstr), 0)) == -1) { Log(" send failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((dr = recv(sd, rbuf, MAXBUFSIZ-1, 0)) == -1) { Log(" recv failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } rbuf[dr] = '\0'; handleHttpRetcode(rbuf); if ((strstr(rbuf, "HTTP/1.1 206")) == NULL) { fprintf(stderr, "Something unhandled happened, shutting down...\n"); exit(1); } s = rbuf; i = 0; while(1 && (s - rbuf) < dr) { if (*s == '\n' && *(s - 1) == '\r' && *(s - 2) == '\n' && *(s - 3) == '\r' && (s - rbuf) < dr) { s++; i++; break; } s++; i++; } if ((dr - i ) > foffset) dw = pwrite(td->fd, s, (foffset - i), td->soffset); else dw = pwrite(td->fd, s, (dr - i), td->soffset); td->offset = td->soffset + dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); pthread_testcancel(); /* Check for pending cancel requests */ while (td->offset < foffset) { fd_set set; /* Set Cancellation Type to Asynchronous * so that a blocking recv() doesn't cause problems */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); /* patch by Gurer Ozen * in slow downloads, avoids excessive CPU consuming... */ FD_ZERO(&set); FD_SET(sd,&set); select(FD_SETSIZE,&set,NULL,NULL,NULL); /* --end of patch */ memset(rbuf, 0, MAXBUFSIZ); dr = recv(sd, rbuf, MAXBUFSIZ, 0); /* Set Cancellation Type back to Deferred */ pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); if ((td->offset + dr) > foffset) dw = pwrite(td->fd, rbuf, foffset - td->offset, td->offset); else dw = pwrite(td->fd, rbuf, dr, td->offset); td->offset += dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); pthread_testcancel(); /* Check for pending cancel requests */ } if (td->offset == td->foffset) td->status = STAT_OK; /* Tell thet download is OK. */ close(sd); pthread_exit(NULL); return NULL; } aget-devel/Download.h0100755000175000000000000000046207611611333014023 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Download.h,v 1.3 2003/01/16 20:15:23 rao Exp $ * */ #ifndef DOWNLOAD_H #define DOWNLOAD_H #include void *http_get(void *); #endif aget-devel/Ftp.c0100755000175000000000000001267410057022515013004 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * * Nearly all Ftp code in Ftp.c, Misc.c and Head.c has been contributed by * Naveen N Rao. */ #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" #include "Ftp.h" extern sigset_t signal_set; extern unsigned int bwritten; extern pthread_mutex_t bwritten_mutex; int parse_pasv_reply2(char *, int, struct sockaddr_in *); int parse_list_reply(char *rbuf, int len); int ftp_get_size(struct sockaddr_in *sin, char *url, int type); void *ftp_get(void *arg) { struct sockaddr_in sin_dc; struct thread_data *td; int sd_c, sd_d; char *sbuf, *rbuf; int dr, dw; long foffset; sigset_t set; pthread_t tid; tid = pthread_self(); /* Block out all signals */ sigfillset(&set); pthread_sigmask(SIG_BLOCK, &set, NULL); /* Set Cancellation Type to Asynchronous */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); td = (struct thread_data *)arg; foffset = td->foffset; if(td->soffset < 0 || td->soffset >= td->foffset) { /* If download complete */ td->status = STAT_OK; /* Tell that download is OK. */ pthread_exit((void *)1); return NULL; } rbuf = (char *)calloc(MAXBUFSIZ, sizeof(char)); sbuf = (char *)calloc(MAXBUFSIZ, sizeof(char)); if (td->head_sd == -1) { if ((sd_c = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log(" socket creation failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((connect(sd_c, (const struct sockaddr *)&td->sin, sizeof(struct sockaddr))) == -1) { Log(" connection failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } /* Should receive manually, cause some servers may not accept any connections! */ if(recv_reply(sd_c, rbuf, MAXBUFSIZ, 0) == -1) { fprintf(stderr, "<<<<<----- %s", rbuf); Log("recv failed: %s", strerror(errno)); pthread_exit((void *)1); } else if(rbuf[0] != '2' && rbuf[0] != '1' && rbuf[0] != '3') { fprintf(stderr, "<<<<<----- %s", rbuf); Log("Seems like the server isn't accepting connections now, bailing out..."); pthread_exit((void *)1); } if(rbuf[0] != '1' && rbuf[0] != '2' && rbuf[0] != '3') { fprintf(stderr, rbuf); handleFTPRetcode(rbuf); close(sd_c); pthread_exit((void *)1); } snprintf(sbuf, MAXBUFSIZ - 1, "USER %s\r\n", td->username); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); if (strncmp(rbuf, "331", 3) != 0) { fprintf(stderr, "Server didnot like username, server said\n"); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } snprintf(sbuf, MAXBUFSIZ - 1,"PASS %s\r\n", td->password); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); if (strncmp(rbuf, "230", 3) != 0) { fprintf(stderr, "Server didnot accept password, server said\n"); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } } else { sd_c = td->head_sd; recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); } snprintf(sbuf, MAXBUFSIZ - 1, "REST %ld\r\n", td->soffset); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); if (strncmp(rbuf, "350", 3) != 0) { fprintf(stderr, "Server doesn't accept resuming transfers."); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } sprintf(sbuf, "TYPE I\r\n"); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); if (strncmp(rbuf, "200", 3) != 0) { fprintf(stderr, "Server didnot accept BINARY transfer.\n"); fprintf(stderr, "Server > %s\n", rbuf); } sprintf(sbuf, "PASV\r\n"); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); recv_reply(sd_c, rbuf, MAXBUFSIZ, 0); parse_pasv_reply2(rbuf, MAXBUFSIZ, &sin_dc); if ((sd_d = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed: %s", strerror(errno)); exit(1); } if ((connect(sd_d, (const struct sockaddr *)&sin_dc, sizeof(sin_dc))) == -1) { Log("Connection failed: %s", strerror(errno)); exit(1); } /* Should avoid preceding '/' */ snprintf(sbuf, MAXBUFSIZ - 1,"RETR %s\r\n", td->url + 1); memset(rbuf, 0, MAXBUFSIZ); send(sd_c, sbuf, strlen(sbuf), 0); td->offset = td->soffset; while (td->offset < foffset) { /* Set Cancellation Type to Asynchronous * in case user needs to cancel recv(). * recv() is a blocking call. */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); memset(rbuf, MAXBUFSIZ, 0); dr = recv_data(sd_d, rbuf, MAXBUFSIZ); /* Set Cancellation Type back to Deferred */ pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); if ((td->offset + dr) > foffset) dw = pwrite(td->fd, rbuf, foffset - td->offset, td->offset); else dw = pwrite(td->fd, rbuf, dr, td->offset); td->offset += dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); pthread_testcancel(); /* Check for pending cancel requests */ } if (td->offset == td->foffset) td->status = STAT_OK; /* Tell that download is OK. */ close(sd_c); close(sd_d); pthread_exit(NULL); return NULL; } aget-devel/Ftp.h0100755000175000000000000000044207611611333013003 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Ftp.h,v 1.3 2003/01/16 20:15:23 rao Exp $ * */ #ifndef FTP_H #define FTP_H #include void *ftp_get(void *); #endif aget-devel/Head.c0100755000175000000000000002662710057023671013123 0ustar muratwheel/* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * */ #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SOLARIS #include #include #endif #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" extern int h_errno; extern char http_proxyhost[VALSIZE]; extern int http_proxyport; int parse_pasv_reply(char *, int, struct sockaddr_in *); int parse_pasv_reply2(char *, int, struct sockaddr_in *); int parse_list_reply(char *rbuf, int len); int ftp_get_size(struct sockaddr_in *sin, char *url, int type); void http_head_req(struct request *req) { struct sockaddr_in sin; struct hostent *he; int sd; char *sbuf; char *rbuf; char *tok; char *s; int clength; char connect_host[1024]; int connect_host_port = 80; int nret = 0; if (strlen(http_proxyhost) > 0) { strncpy(connect_host, http_proxyhost, 1023); connect_host_port = http_proxyport <= 0 ? 80 : http_proxyport; } else { strncpy(connect_host, req->host, 1023); connect_host_port = req->port; } sbuf = (char *)calloc(HEADREQSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(HEADREQSIZ, sizeof(char)); if ((he = gethostbyname(connect_host)) == NULL) { Log("Error: Cannot resolve hostname for %s: %s", req->host, hstrerror(h_errno)); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ-1); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); sin.sin_port = htons(connect_host_port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed for Head Request: %s", strerror(errno)); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { Log("Connection failed for Head Request: %s", strerror(errno)); exit(1); } Log("Head-Request Connection established"); if(strlen(http_proxyhost) > 0) { char tmp[MAXBUFSIZ]; snprintf(tmp, MAXBUFSIZ-1, "http://%s:%d%s", req->host, req->port, req->url); snprintf(sbuf, HEADREQSIZ-1, HEADREQ, tmp, req->host, PROGVERSION); } else { snprintf(sbuf, HEADREQSIZ-1, HEADREQ, req->url, req->host, PROGVERSION); } if ((send(sd, sbuf, strlen(sbuf), 0)) == -1) { Log("send failed for Head Request: %s", strerror(errno)); exit(1); } if ((nret = recv(sd, rbuf, HEADREQSIZ, 0)) == -1) { Log("recv failed for Head Request: %s", strerror(errno)); exit(1); } rbuf[nret] = '\0'; handleHttpRetcode(rbuf); tok = strtok(rbuf, "\r\n"); if ((strstr(tok, "HTTP/1.1 200")) != NULL) { while ((tok = strtok(NULL, "\r\n")) != NULL) { if ((strstr(tok, "Content-Length")) != NULL) { s = (tok + strlen("Content-Length: ")); clength = atoi(s); req->clength = clength; } } } free(sbuf); free(rbuf); } int ftp_head_req(struct request *req, int *head_sd) { struct sockaddr_in sin, sin_dc; struct hostent *he; int sd; char *sbuf; char *rbuf; int resume, ret; sbuf = (char *)calloc(MAXBUFSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(MAXBUFSIZ + 8, sizeof(char)); resume = 0; if ((he = gethostbyname(req->host)) == NULL) { Log("Error: Cannot resolve hostname for %s: %s", req->host, hstrerror(h_errno)); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ-1); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); printf(req->ip); sin.sin_port = htons(req->port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed for Head Request: %s", strerror(errno)); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { Log("Connection failed for Head Request: %s", strerror(errno)); exit(1); } /* Should receive manually, cause some servers may not accept any connections! */ if(recv_reply(sd, rbuf, MAXBUFSIZ, 0) == -1) { Log("recv failed: %s", strerror(errno)); fprintf(stderr, "Server > %s", rbuf); exit(1); } else if(rbuf[0] != '2' && rbuf[0] != '1' && rbuf[0] != '3') { Log("Seems like the server isn't accepting connections now, bailing out..."); fprintf(stderr, "Server > %s", rbuf); exit(1); } fprintf(stderr, "Server > %s", rbuf); rbuf[MAXBUFSIZ -2] = '\0'; if(rbuf[0] != '2' && rbuf[0] != '1' && rbuf[0] != '3') { /* '2' is the usual reply */ handleFTPRetcode(rbuf); exit(1); } Log("Connection established to FTP server, logging in as %s", req->username); snprintf(sbuf, MAXBUFSIZ-1, "USER %s\r\n", req->username); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); if (strncmp(rbuf, "331", 3) != 0) { fprintf(stderr, "Server didnot like username, server said\n"); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } snprintf(sbuf, MAXBUFSIZ-1, "PASS %s\r\n", req->password); printf("Aget > PASS ***********\n"); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); if (strncmp(rbuf, "230", 3) != 0) { fprintf(stderr, "Server didnot accept password, server said\n"); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } Log("Successfully logged into FTP server, checking if server supports resume feature..."); sprintf(sbuf, "REST 150\r\n"); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); if (strncmp(rbuf, "350", 3) != 0) { fprintf(stderr, "Server doesn't accept resuming transfers. I'll use one thread, sorry.\n"); fprintf(stderr, "Server > %s\n", rbuf); resume = 0; exit(1); } resume = 1; sprintf(sbuf, "SYST\r\n"); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); if (strncmp(rbuf, "215", 3) != 0) { fprintf(stderr, "Server didnot return a reply to a SYST command.\n"); fprintf(stderr, "Server > %s\n", rbuf); exit(1); } if (strstr(rbuf, "UNIX") != NULL) req->ftp_server_type = SERVER_TYPE_UNIX; else req->ftp_server_type = SERVER_TYPE_NT; sprintf(sbuf, "TYPE I\r\n"); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); if (strncmp(rbuf, "200", 3) != 0) { fprintf(stderr, "Server didnot accept BINARY transfer.\n"); fprintf(stderr, "Server > %s\n", rbuf); } sprintf(sbuf, "PASV\r\n"); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); parse_pasv_reply2(rbuf, MAXBUFSIZ, &sin_dc); snprintf(sbuf, MAXBUFSIZ -1, "LIST %s\r\n", req->url + 1); printf("Aget > %s", sbuf); memset(rbuf, 0, MAXBUFSIZ); send(sd, sbuf, strlen(sbuf), 0); /* Get LISTing from Data Connection */ if ((req->clength = ftp_get_size(&sin_dc, req->url + 1, req->ftp_server_type)) == -1) { fprintf(stderr, "Couldn't get SIZE for the URL!"); exit(1); } ret = recv_reply(sd, rbuf, MAXBUFSIZ, 0); fprintf(stderr, "Server > %s\n", rbuf); Log("Content length: %d", req->clength); /* close(sd); */ *head_sd = sd; free(sbuf); free(rbuf); return resume; } /* code by naveen n rao */ int parse_pasv_reply(char *rbuf, int len, struct sockaddr_in *sin) { char *s; unsigned int addr[6] = {0, 0, 0, 0, 0, 0}; char dc_addr[MAXIPSIZ]; unsigned short dc_port; int i; int slen; s = strdup(rbuf); slen = strlen(rbuf); s += 4; i = 0; while(!isdigit(*s) && (i++ < slen)) s++; for(i = 0; i < 6; i++) { while((i < 5 && *s != ',' ) || ( i == 5 && *s != ')')) addr[i] = (*(s++) - '0') + (addr[i] * 10); s++; } snprintf(dc_addr, MAXIPSIZ -1, "%d.%d.%d.%d", addr[0], addr[1], addr[2], addr[3]); dc_port = (addr[4] << 8) + addr[5]; #ifdef DEBUG printf("PASV ADDR: %s, PORT: %d\n", dc_addr, dc_port); #endif bzero((char *)sin, sizeof(struct sockaddr_in)); sin->sin_family = AF_INET; sin->sin_port = htons(dc_port); sin->sin_addr.s_addr = inet_addr(dc_addr); return 0; } /* new */ int parse_pasv_reply2(char *rbuf, int len, struct sockaddr_in *sin) { char *s = rbuf; char tmp[4]; unsigned int addr[10] = {0, 0, 0, 0, 0, 0}; char dc_addr[MAXIPSIZ]; unsigned short dc_port; int i = 0, j = 0; for ( ; ((s - rbuf) < len) && (*s != '\0') && (*s != '\r') && (*s != '\n') && (*s != '(');) s++; s++; /* pass '(' */ for (i = 0 ; ((s - rbuf) < len) && (*s != '\0') && (*s != '\r') && (*s != '\n') && (i < 6); s++) { for (j = 0; (s - rbuf < len) && (*s != '\0') && (*s != '\r') && (*s != '\n') && (*s != ',') && (*s != ')') && (j < 3); s++, j++) tmp[j] = *s; tmp[j] = '\0'; addr[i] = atoi(tmp); i++; } if (addr[0] == 0) return -1; snprintf(dc_addr, MAXIPSIZ -1, "%d.%d.%d.%d", addr[0], addr[1], addr[2], addr[3]); dc_port = (addr[4] << 8) + addr[5]; bzero((char *)sin, sizeof(struct sockaddr_in)); sin->sin_family = AF_INET; sin->sin_port = htons(dc_port); sin->sin_addr.s_addr = inet_addr(dc_addr); return 0; } /* by naceen n rao */ int ftp_get_size(struct sockaddr_in *sin, char *url, int type) { int sd; char rbuf[MAXBUFSIZ]; char perm[16]; char attr[16]; char uid[16]; char gid[16]; char size[16]; char month[16]; char day[16]; char year[16]; char file[255]; memset(rbuf, 0x0, MAXBUFSIZ); memset(perm, 0x0, 16); memset(attr, 0x0, 16); memset(uid, 0x0, 16); memset(gid, 0x0, 16); memset(size, 0x0, 16); memset(month, 0x0, 16); memset(day, 0x0, 16); memset(year, 0x0, 16); memset(file, 0x0, 16); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed for FTP Data Connection: %s", strerror(errno)); return -1; } if ((connect(sd, (const struct sockaddr *)sin, sizeof(struct sockaddr))) == -1) { Log("Connection failed FTP Data Connection: %s", strerror(errno)); return -1; } memset(rbuf, 0, MAXBUFSIZ); recv(sd, rbuf, MAXBUFSIZ-1, 0); printf("Server > %s\n", rbuf); if (type == SERVER_TYPE_UNIX) { sscanf(rbuf, "%15s %15s %15s %15s %15s %15s %15s %15s %15s\r\n", perm, attr, uid, gid, size, month, day, year, file); printf("SIZE: %s\n", size); } else { /* -r-xr-xr-x 1 owner group 10060 Apr 12 1999 Lmain.cpp */ sscanf(rbuf, "%15s %15s %15s %15s %15s %15s %15s %15s %15s\r\n", perm, attr, uid, gid, size, month, day, year, file); printf("SIZE: %s\n", size); } close(sd); return atoi(size); } /* by naceen n rao */ int parse_list_reply(char *rbuf, int len) { char *tok; char *tokens[100]; if ((tok = strtok_r(rbuf, "\r\n", tokens)) == NULL) return -1; fprintf(stderr, "Server > %s\n", tok); while ((tok = strtok_r(NULL, "\r\n", tokens)) != NULL) fprintf(stderr, "Server > %s\n", tok); return 0; } aget-devel/Head.h0100755000175000000000000000053607615201054013116 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Head.h,v 1.4 2003/01/27 09:55:56 murat Exp $ * */ #ifndef HEAD_H #define HEAD_H #include "Data.h" void http_head_req(struct request *); int ftp_head_req(struct request *, int *); #endif aget-devel/INSTALL0100755000175000000000000000163007615153011013130 0ustar muratwheelEnderUNIX Aget 0.57 INSTALL -------------------------- The program is tested on RedHat Linux 7.2 (gcc-2.96), FreeBSD 4.5 and Solaris 8. For Solaris, you'll have to use Makefile.Solaris. To compile Aget, type: # make && make strip To install the compiled binary, type: # make install and binary will be installed as /usr/local/bin/aget You're done. All you need is a file to download and supply that url and desired number of segments to aget, e.g: $ aget -n10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3 Aget will split the file into 10 segments and will download it for you. PS: If your OS is MacOS X then, you can use the makefile Makefile.MacOSX If Solaris, you can use Makefile.Solaris, and if it's FreeBSD, you can use the default one or Makefile.FreeBSD. Simply copy the one which suits your platform over Makefile... Fri Nov 22 07:49:15 EET 2002 http://www.enderunix.org/aget/ aget-devel/Makefile0100755000175000000000000000052110040555010013524 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Ftp.o Resume.o loadrc.o CFLAGS = -g -Wall LDFLAGS = -pthread CC = gcc STRIP = strip all: aget aget: $(OBJS) $(CC) -o aget -g $(OBJS) $(LDFLAGS) install: aget cp -f aget /usr/local/bin/aget clean: rm -f *.o aget aget-devel/Makefile.FreeBSD0100755000175000000000000000057407567357333015001 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -Wall -W -pedantic LDFLAGS = -pthread CC = gcc STRIP = strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ c: rm -f *core* *~ *log aget-devel/Makefile.MacOSX0100755000175000000000000000066407567676056014667 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -Wall -W # Remove '-pedantic' to prevent myriad of warnings with gcc3.x LDFLAGS = -lpthread CC = gcc PREFIX = /usr/local all: aget aget: $(OBJS) $(CC) -o $@ $(OBJS) $(LDFLAGS) install: aget install -m 755 -s aget $(PREFIX)/bin/aget clean: rm -f aget *.o core.* *~ c: rm -f *core* *~ *log aget-devel/Makefile.Solaris0100755000175000000000000000061607567357333015200 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -Wall -W -pedantic -DSOLARIS LDFLAGS = -lpthread -lsocket -lnsl -lresolv CC = gcc STRIP=/usr/local/bin/strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ aget-devel/Misc.c0100755000175000000000000002067010057024276013147 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Misc.c,v 1.23 2004/06/01 06:57:02 murat Exp $ * */ #include #include #include #include #include #include #include #include #include #include #include "Misc.h" #include "Data.h" #define VALSIZE 256 extern unsigned int bwritten; /* Used by updateProgressBar() */ extern request *req; extern char ftpanonymoususer[VALSIZE]; extern char ftpanonymouspass[VALSIZE]; extern int preferredthread; extern char http_proxyhost[VALSIZE]; extern char http_proxyuser[VALSIZE]; extern char http_proxypass[VALSIZE]; void parse_url(char *url, struct request *req) { char *s; int i, j, k; i = j = k = 0; s = url; if ((strncmp(url, "https://", 8)) == 0) { fprintf(stderr, "Error: Currently Aget doesn't support HTTPS requests...\n"); exit(1); } else if ((strncmp(url, "ftp://", 6)) == 0) { req->port = (req->port == 0 ? 21 : req->port); req->proto = PROTO_FTP; } else if ((strncmp(url, "http://", 7)) == 0) { req->port = (req->port == 0 ? 80 : req->port); req->proto = PROTO_HTTP; } else { fprintf(stderr, "Error: No protocol specified (http:// or ftp://)...\n"); exit(1); } if ((strncmp(url, "http://", 7)) != 0 && req->proto == PROTO_HTTP) { fprintf(stderr, "Error: URL should be of the form http://...\n"); exit(1); } if ((strncmp(url, "ftp://", 6)) != 0 && req->proto == PROTO_FTP) { fprintf(stderr, "Error: URL should be of the form ftp://...\n"); exit(1); } s = (req->proto == PROTO_HTTP ? url + 7 : url + 6); /* Jump pass http:// or ftp:// part depending on the protocol */ for (i = 0; *s != '/'; i++, s++) { if (i > MAXHOSTSIZ) { fprintf(stderr, "Error: Cannot get hostname from URL...\n"); exit(1); } if (*s == ':') { /* If username:password is supplied: http://naveen:12345@x.y.com/file */ while(*s != '/') { req->username[j++] = *--s; i--; } req->username[--j] = '\0'; revstr(req->username); while(*s != ':') s++; /* Skip past user name --Naveen */ while(1) { if (*s == ':') { while(*s != '@') { if (k > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get password from URL...\n"); exit(1); } req->password[k++] = *++s; } break; } s++; } req->password[--k] = '\0'; } req->host[i] = *s; } req->host[i] = '\0'; for (i = 0; *s != '\0'; i++, s++) { if (i > MAXURLSIZ) { fprintf(stderr, "Error: Cannot get remote file name from URL...\n"); exit(1); } req->url[i] = *s; } req->url[i] = '\0'; --s; for (i = 0; *s != '/'; i++, s--) { if (i > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get local file name from URL...\n"); exit(1); } req->file[i] = *s; } req->file[i] = '\0'; revstr(req->file); /* Required for FTP */ if (strlen(req->username) <= 0) strncpy(req->username, ftpanonymoususer, MAXBUFSIZ - 2); if (strlen(req->password) <= 0) strncpy(req->password, ftpanonymouspass, MAXBUFSIZ - 2); } int numofthreads(int size) { if (size == 0) return 0; else if (size < BUFSIZ * 4) /* < 16384 */ return 1; else if ((size >= BUFSIZ * 4) && (size < BUFSIZ * 8)) /* 16384 < x < 32678 */ return 2; else if ((size >= BUFSIZ * 8) && (size < BUFSIZ * 16)) /* 32768 < x < 65536 */ return 3; else if ((size >= BUFSIZ * 16) && (size < BUFSIZ * 32)) /* 65536 < x < 131072 */ return 4; else if ((size >= BUFSIZ * 32) && (size < BUFSIZ * 64)) /* 131072 < x < 262144 */ return 5; else if ((size >= BUFSIZ * 64) && (size < BUFSIZ * 128)) return 6; else if ((size >= BUFSIZ * 128) && (size < BUFSIZ * 256)) return 7; else if ((size >= BUFSIZ * 256) && (size < BUFSIZ * 512)) return 8; else if ((size >= BUFSIZ * 512) && (size < BUFSIZ * 1024)) return 9; else return 10; } int calc_offset(int total, int part, int nthreads) { return (part * (total / nthreads)); } void usage() { fprintf(stderr, "usage: aget [options] url\n"); fprintf(stderr, "\toptions:\n"); fprintf(stderr, "\t\t-p port number\n"); fprintf(stderr, "\t\t-l local file name\n"); fprintf(stderr, "\t\t-n suggested number of threads\n"); fprintf(stderr, "\t\t-h this screen\n"); fprintf(stderr, "\t\t-v version info\n"); fprintf(stderr, "\n"); fprintf(stderr, "http//www.enderunix.org/aget/\n"); } /* reverse a given string */ void revstr(char *str) { char *p, *s; int i; int size; if ((size = strlen(str)) == 0) return; p = (char *)calloc(size+1, sizeof(char)); s = p; for (i = size; i > 0; i--, s++) *s = *(str + i - 1); *s = '\0'; memset(str, 0, size); strncpy(str, p, size); free(p); } /* Log */ void Log(char *fmt, ...) { va_list ap; char *lfmt; lfmt = (char *)calloc(16 + strlen(fmt), sizeof(char)); sprintf(lfmt, "Aget > %s", fmt); fflush(stdout); va_start(ap, fmt); vfprintf(stderr, lfmt, ap); va_end(ap); if (fmt[0] != '\0' && fmt[strlen(fmt) - 1] == ':') fprintf(stderr, " %s", strerror(errno)); fprintf(stderr, "\n"); free(lfmt); } /* Progress Bar. * Modified to display progress every second using alarm(). * If no progress has been made, nothing is printed for 5 successive calls * i.e., this guarantees display of progress bar every 5 seconds at least * and atmost every one second. */ void updateProgressBar() { float rat; int ndot, i, j; static float prev = 0; char fmt[128]; float cur = (float)bwritten; float tot = (float)req->clength; rat = cur/tot; ndot = (int)(rat * 100); if(ndot > 100) ndot = 100; for (i = 0; i < ndot; i += 2) putchar('='); putchar('>'); i += 2; for (i = ndot - 1; i < 100; i += 2) putchar(' '); sprintf(fmt, "[%.2d%% done] Bytes: %u", ndot, bwritten); i += strlen(fmt); prev = ndot; printf("%s", fmt); /* Progress Bar should stay on only ONE line! */ for (j = 0; j < i; j++) putchar(0x08); fflush(stdout); } void handleHttpRetcode(char *rbuf) { if ((strstr(rbuf, "HTTP/1.1 416")) != NULL) { Log("Server returned HTTP/1.1 416 - Requested Range Not Satisfiable\n"); exit(1); } else if ((strstr(rbuf, "HTTP/1.1 403")) != NULL) { Log(" 0) { if(max--) goto retry; } } return nrecv; } /* Should be called only on error reply to display a suitable message */ void handleFTPRetcode(char *rbuf) { if(rbuf[0] == '4') { if(rbuf[1] == '2') Log("There seems to be an error in the connections."); else if(rbuf[1] == '5') Log("Requested action not taken"); Log("Retry the download after sometime."); Log("If the error persists, please report this with the command line used to murat@enderunix.org."); } else if(rbuf[0] == '5') { if(rbuf[1] == '0') Log("The server has reported a syntax error."); else if(rbuf[1] == '3') Log("Error relating to login; checkout the previous message, if available."); else if(rbuf[1] == '5') Log("Requested action not taken"); Log("If you feel this is an error with aget, please report this with the command line used to murat@enderunix.org."); } } /* This can handle multiline replies in FTP protocol by naveen n rao */ int recv_reply(int sd, char *rbuf, int len, int flags) { char line[1024]; FILE *fp; int nread; int done; char status[4] = {0}; if ((fp = fdopen(sd, "r")) == NULL) return -1; nread = 0; done = 0; do { fgets(line, 1020, fp); if (line[3] == '-' && status[0] == 0) strncpy(status, line, 3); if (line[3] == ' ') { if (status[0] != 0) { if (strncmp(status, line, 3) == 0) done = 1; } else done = 1; } memcpy(rbuf + nread, line, strlen(line)); nread += strlen(line); } while (done != 1 && nread < 1022); #ifdef DEBUG printf("READ: %d\n", nread); fwrite(rbuf, 1, nread, stderr); #endif return nread; } aget-devel/Misc.h0100755000175000000000000000144107615160051013145 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Misc.h,v 1.8 2003/01/27 07:30:49 murat Exp $ * */ #ifndef MISC_H #define MISC_H #include "Data.h" #define LOGSIZ 1024 int calc_offset(int, int, int); int numofthreads(int); void parse_url(char *, struct request *); void usage(); void revstr(char *); /* Reverse String */ void Log(char *, ...); /* Log */ void updateProgressBar(); void handleHttpRetcode(char *); void sendCmd(int, char *, int, char *, int); void recvRep(int, char *, int, int); int recv_data(int, char *, int); int process(char *, char *, int *, int *, int *, int *, int); void handleFTPRetcode(char *rbuf); int recv_reply(int, char *, int, int); #endif aget-devel/README0100755000175000000000000000075507615153011012766 0ustar muratwheelEnderUNIX Aget 0.57 README -------------------------- This program is a starting point for a very useful project like FlashGet for Win32. My aim is to provide all the functionality that program has. If you need assistance in installing Aget, you can see INSTALL file, Program is tested on RedHat Linux 8, FreeBSD 4.7 and Solaris 7,8 (Sparc), MacOSX. It proved to be successful. You can get much more info from the Aget Web site: http://www.enderunix.org/aget Fri Nov 22 07:50:38 EET 2002 aget-devel/README-Developer0100755000175000000000000000307207605324460014713 0ustar muratwheelOk, Here's a developer's primer for aget source: The code is splitted into several files: main.{c|h} * starts a thread for handling signals. * gets program options and decides the program's behviour by setting a few global variables. Aget.{c|h} * get() and resume_get() functions starts the download process. Depending on the protocol, necessary methods are called. resume_get() is for resume jobs. If the there is a log file of a previous job, the values are taken from it. Head.{c|h} * Before the actual download, a head request is sent to the server to determine the current status of the file. Some values like: 1. HTTP return code (e.g. 404, 206, 403, 200) 2. Content-length might be of interest there. Ftp.{c|h} * ftp_get() function which is the thread runner for ftp downloads. Download.{c|h} * Aget is no more %100 Posix compliant :( pwrite is from X_OPEN standarts. I had to do it, since I needed a way to simultaneously write to the same file without loosing any time while synchronising the threads. For the time, there is only http_get. When aget is ftp-aware ftp_get will be in this file. Resume.{c|h} * Methods about saving and restoring history files are in this file. Signal.{c|h} * Signal Handling functions. Misc.{c|h} * Other functions which do not fit any of the general categories are stored here. These are generally helper functions... For more information read the aget source, and you may contact murat at enderunix dot org. Fri Nov 22 07:51:04 EET 2002 aget-devel/Resume.c0100755000175000000000000000756207611611333013517 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Resume.c,v 1.7 2003/01/16 20:15:23 rao Exp $ * */ #include #include #include #include #include #include #include #include "Data.h" #include "Resume.h" #include "Misc.h" extern struct thread_data *wthread; extern struct request *req; extern int nthreads; extern int bwritten; extern time_t t_start, t_finish; void save_log() { char *logfile, *s; struct hist_data h; FILE *fp; time_t diff_sec; struct passwd *pw; if(wthread == NULL) // Do nothing return; if(bwritten == req->clength) { // If download has just completed time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download already complete: job completed in %d seconds. (%d Kb/sec)", diff_sec, (req->clength / diff_sec) / 1024); Log("Shutting down..."); return; } logfile = (char *)calloc(255, sizeof(char)); /* Try and save log file in user's home directory */ pw = getpwuid(getuid()); if(pw->pw_dir == NULL) /* A small fix: save protocol in log filename. * This is necessary in case urls for ftp and http locations of * a file is the same -- rare, but happens to me all the time (localhost) */ snprintf(logfile, 255, "%s-aget%c.log", req[0].lfile, req->proto == PROTO_HTTP ? 'h' : 'f'); else { s = &req->lfile[strlen(req->lfile)-1]; while(*s!='/' && s!=req->lfile) s--; snprintf(logfile, 255, "%s%s-aget%c.log", pw->pw_dir, s, req->proto == PROTO_HTTP ? 'h' : 'f'); } if ((fp = fopen(logfile, "w")) == NULL) { fprintf(stderr, "Cannot open log file %s for writing: %s\n", logfile, strerror(errno)); exit(1); } /* Get the finish time, derive some stats */ time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ req->time_taken += diff_sec; memcpy(&(h.req), req, sizeof(struct request)); memcpy(&(h.wthread), wthread, sizeof(struct thread_data) * nthreads); h.nthreads = nthreads; h.bwritten = bwritten; printf("--> Logfile is: %s, so far %d bytes have been transferred\n", logfile, h.bwritten); fwrite(&h, sizeof(struct hist_data), 1, fp); fclose(fp); free(logfile); } int read_log(struct hist_data *h) { char *logfile, *s; FILE *fp; struct passwd *pw; logfile = (char *)calloc(255, sizeof(char)); /* First see if the log file is in present directory, if not check user's home directory */ snprintf(logfile, 255, "%s-aget%c.log", req[0].lfile, req->proto == PROTO_HTTP ? 'h' : 'f'); Log("Attempting to read log file for resuming download job..."); if ((fp = fopen(logfile, "r")) == NULL) { if (errno == ENOENT) { pw = getpwuid(getuid()); if(pw->pw_dir != NULL) { s = &req->lfile[strlen(req->lfile)-1]; while(*s!='/' && s!=req->lfile) s--; snprintf(logfile, 255, "%s%s-aget%c.log", pw->pw_dir, s, req->proto == PROTO_HTTP ? 'h' : 'f'); if ((fp = fopen(logfile, "r")) == NULL) { if (errno == ENOENT) { Log("Couldn't find log file for this download, starting a clean job..."); return -1; } else { fprintf(stderr, "Cannot open log file %s for reading: %s\n", logfile, strerror(errno)); exit(1); } } } else { Log("Couldn't find log file for this download, starting a clean job..."); return -1; } } else { fprintf(stderr, "Cannot open log file %s for reading: %s\n", logfile, strerror(errno)); exit(1); } } fread(h, sizeof(struct hist_data), 1, fp); bwritten = h->bwritten; fclose(fp); Log("%d bytes already transferred", bwritten); /* Unlinking logfile after we've read it. */ if ((unlink(logfile)) == -1) fprintf(stderr, "read_log: cannot remove stale log file %s: %s\n", logfile, strerror(errno)); free(logfile); return 0; } aget-devel/Resume.h0100755000175000000000000000073307611611333013515 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Resume.h,v 1.4 2003/01/16 20:15:23 rao Exp $ * */ #ifndef RESUME_H #define RESUME_H #include "Data.h" #include "Defs.h" typedef struct hist_data { struct request req; int nthreads; int bwritten; struct thread_data wthread[MAXTHREADS]; } hist_data; void save_log(); int read_log(struct hist_data *); #endif aget-devel/Signal.c0100755000175000000000000000353310040555010013453 0ustar muratwheel/* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Signal.c,v 1.5 2004/04/18 19:42:00 murat Exp $ * */ #include #include #include #include #include #include "Signal.h" #include "Data.h" #include "Resume.h" #include "Misc.h" extern int nthreads; extern struct thread_data *wthread; extern struct request *req; extern int bwritten; extern pthread_mutex_t bwritten_mutex; extern pthread_t main_tid; void *signal_waiter(void *arg) { int signal; sigset_t set; arg = NULL; /* Set Cancellation Type to Asynchronous */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); sigfillset(&set); pthread_sigmask(SIG_BLOCK, &set, NULL); sigaddset(&set, SIGINT); sigaddset(&set, SIGTERM); sigaddset(&set, SIGALRM); /* Set off alarm so that progressbar is updated at regular intervals */ alarm(1); while(1) { /* TODO for Solaris, fix bugs here. */ #ifdef SOLARIS sigwait(&set); #else sigwait(&set, &signal); #endif switch(signal) { case SIGINT: sigint_handler(); break; case SIGALRM: sigalrm_handler(); break; } } } void sigint_handler(void) { int i; fflush(stdout); printf("^C caught, terminating download job. Please wait...\n"); /* Cancel main thread - can cause serious problems otherwise. * This is because, we will be restarting threads which are interrupted * or which terminate in error in Aget.c */ pthread_cancel(main_tid); pthread_join(main_tid, NULL); for (i = 0; i < nthreads; i++) { pthread_cancel(wthread[i].tid); wthread[i].status &= STAT_INT; /* Interrupted download */ } printf("Download job terminated. Now saving download job...\n"); save_log(); exit(0); } void sigalrm_handler(void) { updateProgressBar(); alarm(1); } aget-devel/Signal.h0100755000175000000000000000057507614772647013521 0ustar muratwheel/* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: Signal.h,v 1.3 2003/01/26 14:50:15 murat Exp $ * */ #ifndef SIGNAL_H #define SIGNAL_H #include #include void *signal_waiter(void *); void sigint_handler(void); void sigalrm_handler(void); #endif aget-devel/THANKS0100755000175000000000000000131310002160532012776 0ustar muratwheelAaron Zauner For submitting aget into FreeBSD ports tree. Gurer Ozen Patrick Marcae Naveen N. Rao for the initial FTP support. He gave ftp support in the middle of his exams ;) Sensei, and my EnderUNIX team members... Atilim Boy nikinakof Oleg Mitsura Aubin Paul MacOSX patch. Stefano Balocco Aget RPM Bruno Barrera C. Debian Packages Dries Verachtert Aget RPM for Fedora Core Fri Nov 22 07:51:36 EET 2002 aget-devel/TODO0100755000175000000000000000061307615561767012614 0ustar muratwheelEnderUNIX Aget 0.57 TODO: ------------------------- KNOWN BUGS: * Resume code is still buggy on Solaris and BSD systems. * Logging out from FTP servers is not so graceful. * HTTP Proxy authentication * FreeBSD/NetBSD/OpenBSD Port/Package * Multiple downloads * Constructing libaget.so/libaget.a for IstanbulX project of acikkod.org Fri Nov 22 07:55:26 EET 2002 http://www.enderunix.org/aget/ aget-devel/loadrc.c0100755000175000000000000000671010057022515013511 0ustar muratwheel#include #include #include #include #include #include #include #include #include #include #include "loadrc.h" extern int errno; void loadconfig(char *cfgfile) /* load aget rc file */ { FILE *fd; char buf[BUFSIZE]; char keyword[KEYSIZE]; char value[VALSIZE]; char *cp1, *cp2; char *variables[] = { "Invalid", "ftpanonymoususer", "ftpanonymouspass", "http_proxyhost", "http_proxyuser", "http_proxypass", "preferredthread" }; int i, key, line, keyword_nums = sizeof(variables)/sizeof(char *); int slen = 0; int j = 0; if ((fd = fopen(cfgfile, "r")) == NULL) { fprintf(stderr, "loadconfig: cannot open aget rc file %s, exiting...\n", cfgfile); exit(-1); } line = 0; i = 0; while ((fgets(buf, BUFSIZE, fd)) != NULL) { line++; if (buf[0] == '#') continue; if ((slen = strlen(buf)) <= 1) continue; cp1 = buf; cp2 = keyword; while (isspace((int)*cp1) && (cp1 - buf < slen)) cp1++; j = 0; while (isgraph((int)*cp1) && (*cp1 != '=') && (j++ < KEYSIZE -2) && (cp1 - buf < slen)) *cp2++ = *cp1++; j = 0; *cp2 = '\0'; cp2 = value; while ((*cp1 != '\0') && (*cp1 !='\n') && (*cp1 !='=') && (cp1 - buf < slen)) cp1++; cp1++; while (isspace((int)*cp1) && (cp1 - buf < slen)) cp1++; if (*cp1 == '"') cp1++; while ((*cp1 != '\0') && (*cp1 !='\n') && (*cp1 !='"') && (j++ < VALSIZE -2) && (cp1 - buf < slen)) *cp2++ = *cp1++; *cp2-- = '\0'; if (keyword[0] =='\0' || value[0] =='\0') continue; key = 0; for (i = 0; i < keyword_nums; i++) { if ((strcmp(keyword, variables[i])) == 0) { key = i; break; } } switch(key) { case 0: fprintf(stderr, "Illegal Keyword in RC file %s: %s\n", cfgfile, keyword); break; case 1: strncpy(ftpanonymoususer, value, VALSIZE); break; case 2: strncpy(ftpanonymouspass, value, VALSIZE); break; case 3: strncpy(http_proxyhost, value, VALSIZE); break; case 4: strncpy(http_proxyuser, value, VALSIZE); break; case 5: strncpy(http_proxypass, value, VALSIZE); break; case 6: preferredthread = atoi(value); break; } } fclose(fd); } void createrc(char *f) { FILE *fp; if ((fp = fopen(f, "w")) == NULL) { fprintf(stderr, "Aget: couldn't create agetrc file %s: %s\n", f, strerror(errno)); return; } fprintf(fp, "ftpanonymoususer = ftp\n"); fprintf(fp, "ftpanonymouspass = aget@enderunix.org\n"); fclose(fp); } void readrc() { struct passwd *pw; struct stat st; char agetrc[1024]; char *tok; if ((pw = getpwuid(getuid())) == NULL) return; snprintf(agetrc, 1020, "%s/.agetrc", pw->pw_dir); if ((stat(agetrc, &st)) == -1) { if (errno == ENOENT) createrc(agetrc); else { fprintf(stderr, "cannot read agetrc file %s: %s. Might be corrupt!\n", agetrc, strerror(errno)); exit(1); } } else loadconfig(agetrc); if (strlen(http_proxyhost) > 0) { if ((tok = strtok(http_proxyhost, ":")) != NULL) { strncpy(http_proxyhost, tok, VALSIZE -1); tok = strtok(NULL, ":"); if(tok != NULL) { http_proxyport = atoi(tok); } else { http_proxyport = 80; } } } #ifdef DEBUG printf("ftpanon: %s\n", ftpanonymoususer); printf("ftpanonpa: %s\n", ftpanonymouspass); printf("ph: %s\n", http_proxyhost); printf("pu: %s\n", http_proxyuser); printf("ppass: %s\n", http_proxypass); printf("pport: %d\n", http_proxyport); printf("pt = %d\n", preferredthread); #endif } aget-devel/loadrc.h0100755000175000000000000000067407615562542013540 0ustar muratwheel#ifndef LOADCONFIG_H #define LOADCONFIG_H #include #define TOKENS " \n" enum { BUFSIZE = 1024, KEYSIZE = 64, VALSIZE = 256 }; char ftpanonymoususer[VALSIZE]; char ftpanonymouspass[VALSIZE]; int preferredthread = -1; char http_proxyhost[VALSIZE]; int http_proxyport = -1; char http_proxyuser[VALSIZE]; char http_proxypass[VALSIZE]; void loadconfig(char *); void readrc(); void removespaces(char *); #endif aget-devel/main.c0100755000175000000000000001072610057022515013173 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: main.c,v 1.14 2004/06/01 06:42:21 murat Exp $ * */ #include #include #include #include #include #include #include #include "Defs.h" #include "Data.h" #include "Misc.h" #include "Aget.h" #include "Signal.h" #include "Resume.h" #define VALSIZE 256 extern char ftpanonymoususer[VALSIZE]; extern char ftpanonymouspass[VALSIZE]; extern int preferredthread; extern char http_proxyhost[VALSIZE]; extern char http_proxyuser[VALSIZE]; extern char http_proxypass[VALSIZE]; extern int http_proxyport; extern void readrc(); int nthreads; int fsuggested; struct request *req; /* Download jobs */ pthread_t hthread; /* Helper thread for signals */ struct thread_data *wthread; /* Worker Threads */ pthread_t main_tid; int main(int argc, char **argv) { int c, error = 0; struct hist_data h; int retlog; char tmp[MAXBUFSIZ]; char *fullurl; extern char *optarg; extern int optind; sigset_t set; main_tid = pthread_self(); /* Allocate heap for download request * struct request stores all the information that might be * of interest */ req = (struct request *)calloc(1, sizeof(struct request)); /* This is required because if download is aborted before * this is initialized, problems will occur */ wthread = NULL; /* except from signal handler thread, * no other thread is receiving signals! * No signals are caught here since we need to save download job only * after wthread structures are initialized. * TODO:: We need to create helper thread as soon as head request is over * since the user might wish to stop downloading as soon as he sees the * content length. */ sigfillset(&set); pthread_sigmask(SIG_BLOCK, &set, NULL); /* Safe to exit before download starts. * This is set back to DEFERRED before creating helper thread because * we want this to exit only at certain safe points specified using * pthread_testcancel(). * Helper thread is started just before creating worker threads and after * wthread structures are initialized. Only after this point do we need to * save download job if an interrupt comes - not before. Helper thread exists * only to catch SIGINT and to save download job. It doesn't make sense to * start it before wthread structures are initialized! */ pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); /* Read the RC file if exist */ readrc(); /* If set in rc file */ if (preferredthread != -1) nthreads = preferredthread; while (!error && (c = getopt(argc,argv,"p:l:n:hfv")) != -1) { switch(c) { case 'p': req->port = atoi(optarg); break; case 'l': strncpy(req->lfile, optarg, MAXBUFSIZ -1); break; case 'n': fsuggested = 1; if ((nthreads = atoi(optarg)) > MAXTHREADS) { Log("Error: Maximum # of threads allowed is %d\n", MAXTHREADS); exit(1); } break; case 'h': printf("%s\n", PROGVERSION); usage(); exit(0); break; case 'v': printf("%s\nby Murat BALABAN \n", PROGVERSION); exit(0); break; default: error = 1; usage(); exit(1); break; } } if (error) { usage(); exit(1); } if (fsuggested == 1 && nthreads == 0) { fprintf(stderr, "ERROR: -f and -n should be used together!, exiting...\n"); usage(); exit(1); } if (argc == 2) /* If only url is supplied... */ fullurl = strdup(argv[1]); else if (optind < argc) if (argc > 2) fullurl = strdup(argv[optind]); else { usage(); exit(1); } else if (optind == argc) { usage(); exit(1); } parse_url(fullurl, req); if(strlen(req->lfile) == 0) strncpy(req->lfile, req->file, MAXBUFSIZ - 2); getcwd(tmp, MAXBUFSIZ -2); strncpy(tmp+strlen(tmp), "/", MAXBUFSIZ - strlen(tmp) - 2); strncpy(tmp+strlen(tmp), req->lfile, MAXBUFSIZ - strlen(tmp) - 3); strncpy(req->lfile, tmp, MAXBUFSIZ - 2); /* If a log file for a previous try has been found, read it and * resume the download job (resume_get), otherwise, start with * a clean job (get) * * Logfile is of the pattern: aget-$file_name.log */ switch(req->proto) { case PROTO_HTTP: if ((retlog = read_log(&h)) != -1) resumeDownload(&h, PROTO_HTTP); else startHTTP(req); break; case PROTO_FTP: if ((retlog = read_log(&h)) != -1) resumeDownload(&h, PROTO_FTP); else startFTP(req); break; } return 0; } aget-devel/main.h0100755000175000000000000000074707603413755013217 0ustar muratwheel /* * Aget, Multithreaded Download Accelerator * * (c) 2002 Murat Balaban * See COPYING for copyright information * * $Id: main.h,v 1.3 2002/12/28 21:30:21 murat Exp $ * */ #ifndef MAIN_H #define MAIN_H #include #include "Data.h" char *fullurl; int nthreads; int fsuggested = 0; struct request *req; /* Download jobs */ pthread_t hthread; /* Helper thread for signals */ struct thread_data *wthread; /* Worker Threads */ #endif