aget-0.4.1/0000755000076400000120000000000011202734600010705 5ustar mbwheelaget-0.4.1/Download.c0000644000076400000120000000555111202204557012631 0ustar mbwheel #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" #include "Download.h" extern sigset_t signal_set; unsigned int bwritten = 0; pthread_mutex_t bwritten_mutex = PTHREAD_MUTEX_INITIALIZER; void * http_get(void *arg) { struct thread_data *td; int sd; char *rbuf, *s; long long dr, dw, i; long long foffset; pthread_t tid; tid = pthread_self(); /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &signal_set, NULL); /* Set Cancellation Type to Asynchronous */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); td = (struct thread_data *)arg; foffset = td->foffset; rbuf = (char *)calloc(GETRECVSIZ, sizeof(char)); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log(" socket creation failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((connect(sd, (const struct sockaddr *)&td->sin, sizeof(struct sockaddr))) == -1) { Log(" connection failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((send(sd, td->getstr, strlen(td->getstr), 0)) == -1) { Log(" send failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((dr = recv(sd, rbuf, GETRECVSIZ, 0)) == -1) { Log(" recv failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } handleHttpRetcode(rbuf); if ((strstr(rbuf, "HTTP/1.1 206")) == NULL) { fprintf(stderr, "Something unhandled happened, shutting down...\n"); exit(1); } s = rbuf; i = 0; while(1) { if (*s == '\n' && *(s - 1) == '\r' && *(s - 2) == '\n' && *(s - 3) == '\r') { s++; i++; break; } s++; i++; } td->offset = td->soffset; if ((dr - i ) > foffset) dw = pwrite(td->fd, s, (foffset - i), td->soffset); else dw = pwrite(td->fd, s, (dr - i), td->soffset); td->offset = td->soffset + dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); while (td->offset < foffset) { memset(rbuf, GETRECVSIZ, 0); dr = recv(sd, rbuf, GETRECVSIZ, 0); if ((td->offset + dr) > foffset) dw = pwrite(td->fd, rbuf, foffset - td->offset, td->offset); else dw = pwrite(td->fd, rbuf, dr, td->offset); td->offset += dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); updateProgressBar(bwritten, td->clength); } if (td->offset == td->foffset) td->status = STAT_OK; /* Tell thet download is OK. */ close(sd); /* printf(" Part %d completed, leaving thread...\n", tid, td->tind);*/ pthread_exit(NULL); return NULL; } aget-0.4.1/Head.c0000644000076400000120000000443411202203642011714 0ustar mbwheel#ifndef SOLARIS #define _XOPEN_SOURCE 500 #endif #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SOLARIS #include #include #endif #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" #include extern int errno; extern int h_errno; #ifdef SOLARIS extern const char *hstrerror(int); #endif void http_head_req(struct request *req) { struct sockaddr_in sin; struct hostent *he; int sd; char *sbuf; char *rbuf; char *tok; char *s; long long clength; sbuf = (char *)calloc(HEADREQSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(HEADREQSIZ, sizeof(char)); if ((he = gethostbyname(req->host)) == NULL) { Log("Error: Cannot resolve hostname for %s: %s", req->host, hstrerror(h_errno)); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ); time(&t_start); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); sin.sin_port = htons(req->port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed for Head Request: %s", strerror(errno)); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { Log("Connection failed for Head Request: %s", strerror(errno)); exit(1); } Log("Head-Request Connection established"); sprintf(sbuf, HEADREQ, req->url, req->host, PROGVERSION); if ((send(sd, sbuf, strlen(sbuf), 0)) == -1) { Log("send failed for Head Request: %s", strerror(errno)); exit(1); } if ((recv(sd, rbuf, HEADREQSIZ, 0)) == -1) { Log("recv failed for Head Request: %s", strerror(errno)); exit(1); } handleHttpRetcode(rbuf); tok = strtok(rbuf, "\r\n"); if ((strstr(tok, "HTTP/1.1 200")) != NULL) { while ((tok = strtok(NULL, "\r\n")) != NULL) { if ((strncasecmp(tok, "Content-Length:", 15)) == 0 ) { s = (tok + strlen("Content-Length: ")); clength = atoll(s); req->clength = clength; } } } free(sbuf); free(rbuf); } aget-0.4.1/Misc.h0000644000076400000120000000105311202204002011734 0ustar mbwheel /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #ifndef MISC_H #define MISC_H #include "Data.h" #define LOGSIZ 1024 int calc_offset(long long, int, int); int numofthreads(int); void parse_url(char *, struct request *); void usage(); void revstr(char *); /* Reverse String */ void Log(char *, ...); /* Log */ void updateProgressBar(float, float); void handleHttpRetcode(char *); time_t t_start, t_finish; #endif aget-0.4.1/aget.1.Solaris0000644000076400000120000000405311202454473013333 0ustar mbwheel.TH aget 1 "23 Feb 2009" "SunOS 5.11" "User Commands" .SH NAME Aget \- multi-threaded HTTP download accelerator .SH SYNOPSIS .LP .nf \fBaget\fR [\fBoptions\fR]... \fI\s-1URL\s0\fR .fi .SH DESCRIPTION .LP Aget is a multi-threaded HTTP download accelerator that is utilizing the POSIX thread library - \fBlibpthread\fR(3LIB). Aget is an acronym for two Turkish words "Acele Getir" (English: Get it fast!). Tests show that Aget is successful in realizing its objectives. A file of size 36,347,010 bytes was downloaded in 14 minutes 28 secs via \fBwget\fR(1); whereas it was downloaded in 3 minutes and 15 seconds via aget. If downloading a file of size less than 512K bytes, it's suggested to use small segments, i.e. 4-5. However for large files, it's urged to increase the number of threads to speed up. .SH OPTIONS .sp .ne 2 .mk .na \fB-p\fR \fIport\fR .ad .RS 15n .rt \fIport\fR number to connect to. .RE .sp .ne 2 .mk .na \fB-l\fR \fIfile\fR .ad .RS 15n .rt local \fIfile\fR name, if different from the remote file. .RE .sp .ne 2 .mk .na \fB-n\fR \fIthreads\fR .ad .RS 15n .rt suggested number of \fIthreads\fR. If no thread number is supplied, it is defaulted to 1. .RE .sp .ne 2 .mk .na \fB-f\fR .ad .RS 15n .rt force use of suggested number of \fIthreads\fR. .RE .sp .ne 2 .mk .na \fB-h\fR .ad .RS 15n .rt help screen. .RE .sp .ne 2 .mk .na \fB-v\fR .ad .RS 15n .rt version info. .RE .SH EXIT STATUS .LP The following exit values are returned: .sp .ne 2 .mk .na \fB\fB0\fR \fR .ad .RS 7n .rt Successful completion. .RE .sp .ne 2 .mk .na \fB\fB1\fR \fR .ad .RS 7n .rt Application exited with failure. .RE .sp .ne 2 .mk .na \fB\fB-1\fR \fR .ad .RS 7n .rt POSIX pthreads creation error occurred. .RE .SH "EXAMPLES" .PP \fBExample 1: Say you want to download a \fIURL\fR using 10 \fIthreads\fR, just type:\fR .PP .PP .nf example% \fBaget -n 10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3\fR .fi .SH SEE ALSO .LP wget(1), libpthread(3LIB) .SH BUGS .LP This program does not support IPv6 yet, this is an architectural bug that will be fixed. aget-0.4.1/Signal.h0000644000076400000120000000030307566442007012306 0ustar mbwheel#ifndef SIGNAL_H #define SIGNAL_H #include #include sigset_t signal_set; void * signal_waiter(void *arg); void sigint_handler(void); void sigalrm_handler(void); #endif aget-0.4.1/Makefile.FreeBSD0000644000076400000120000000061611202456250013563 0ustar mbwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -W LDFLAGS = -pthread CC = gcc STRIP = strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget cp -f aget.1 /usr/share/man/man1/ clean: rm -f aget *.o core.* *~ c: rm -f *core* *~ *log aget-0.4.1/Data.h0000644000076400000120000000152711202203762011734 0ustar mbwheel#ifndef DATA_H #define DATA_H #include #include #include "Defs.h" typedef struct request { char host[MAXHOSTSIZ]; /* Remote host */ char url[MAXURLSIZ]; /* URL */ char file[MAXBUFSIZ]; /* file name */ char lfile[MAXBUFSIZ]; /* if local file name is specified */ char ip[MAXIPSIZ]; /* Remote IP */ char username[MAXBUFSIZ]; char password[MAXBUFSIZ]; int port; long long clength; /* Content-length */ unsigned char proto; /* Protocol */ } request; typedef struct thread_data { struct sockaddr_in sin; char getstr[GETREQSIZ]; long long soffset; /* Start offset */ long long foffset; /* Finish offset */ long long offset; /* Current Offset */ long long clength; /* Content Length */ int fd; pthread_t tid; /* Thread ID */ unsigned char status; /* thread exit status */ } thread_data; #endif aget-0.4.1/Aget.c0000644000076400000120000001247111202205536011737 0ustar mbwheel #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Aget.h" #include "Misc.h" #include "Download.h" #include "Resume.h" #include "Data.h" extern struct thread_data *wthread; extern struct request *req; extern int fsuggested, nthreads; extern int bwritten; extern pthread_t hthread; #include extern int errno; void get(struct request *req) { int i, ret, fd, diff_sec, nok = 0; long long soffset, foffset; char *fmt; if (req->proto == PROTO_HTTP) http_head_req(req); /* According to the content-length, get the * suggested number of threads to open. * if the user insists on his value, let it be that way, * use the user's value. */ ret = numofthreads(req->clength); if (fsuggested == 0) { if (ret == 0) nthreads = 1; else nthreads = ret; } wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data)); Log("Downloading %s (%lld bytes) from site %s(%s:%d). Number of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if (strlen(req->lfile) != 0) { if ((fd = open(req->lfile, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else { if ((fd = open(req->file, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } if ((lseek(fd, req->clength - 1, SEEK_SET)) == -1) { fprintf(stderr, "get: couldn't lseek: %s\n", strerror(errno)); exit(1); } if ((write(fd, "0", 1)) == -1) { fprintf(stderr, "get: couldn't allocate space for download file: %s\n", strerror(errno)); exit(1); } /* Get the starting time, prepare GET format string, and start the threads */ fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); time(&t_start); for (i = 0; i < nthreads; i++) { soffset = calc_offset(req->clength, i, nthreads); foffset = calc_offset(req->clength, i + 1, nthreads); wthread[i].soffset = soffset; wthread[i].foffset = (i == nthreads - 1 ? req->clength : foffset); wthread[i].sin.sin_family = AF_INET; wthread[i].sin.sin_addr.s_addr = inet_addr(req->ip); wthread[i].sin.sin_port = htons(req->port); wthread[i].fd = dup(fd); wthread[i].clength = req->clength; snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, soffset); strncpy(wthread[i].getstr, fmt, GETREQSIZ); pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])); } free(fmt); /* Wait for all of the threads to finish... * * TODO: If a thread fails, restart that! */ for (i = 0; i < nthreads; i++) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } if (nok == nthreads) pthread_cancel(hthread); else pthread_join(hthread, NULL); /* Get the finish time, derive some stats */ time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%lld Kb/sec)", diff_sec, (req->clength / diff_sec) / 1024); Log("Shutting down..."); close(fd); } void resume_get(struct hist_data *h) { int i, fd, diff_sec, nok = 0; char *fmt; nthreads = h->nthreads; fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data)); memcpy(req, &h->req, sizeof(struct request)); memcpy(wthread, h->wthread, sizeof(struct thread_data) * nthreads); Log("Resuming download %s (%lld bytes) from site %s(%s:%d). Number of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if (strlen(req->lfile) != 0) { if ((fd = open(req->lfile, O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else { if ((fd = open(req->file, O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } time(&t_start); #ifdef DEBUG for (i = 0; i < nthreads; i++) printf("Start: %lld, Finish: %lld, Offset: %lld, Diff: %lld\n", wthread[i].soffset, wthread[i].foffset, wthread[i].offset, wthread[i].offset - wthread[i].soffset); #endif for (i = 0; i < nthreads; i++) { wthread[i].soffset = wthread[i].offset; wthread[i].fd = dup(fd); snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, wthread[i].offset); strncpy(wthread[i].getstr, fmt, GETREQSIZ); pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])); } for (i = 0; i < nthreads; i++) pthread_join(wthread[i].tid, NULL); for (i = 0; i < nthreads; i++) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } if (nok == nthreads) pthread_cancel(hthread); else pthread_join(hthread, NULL); time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%lld Kb/sec)", diff_sec, ((req->clength - h->bwritten) / diff_sec) / 1024); Log("Shutting down..."); close(fd); } aget-0.4.1/main.h0000644000076400000120000000042507566442007012022 0ustar mbwheel#ifndef MAIN_H #define MAIN_H #include #include "Data.h" char *fullurl; int nthreads; int fsuggested = 0; struct request *req; /* Download jobs */ pthread_t hthread; /* Helper thread for signals */ struct thread_data *wthread; /* Worker Threads */ #endif aget-0.4.1/Signal.c0000644000076400000120000000201407567357333012312 0ustar mbwheel #include #include #include #include #include #include "Signal.h" #include "Data.h" #include "Resume.h" #include "Misc.h" extern int nthreads; extern struct thread_data *wthread; extern struct request *req; extern int bwritten; extern pthread_mutex_t bwritten_mutex; void * signal_waiter(void *arg) { int signal; arg = NULL; pthread_sigmask(SIG_UNBLOCK, &signal_set, NULL); while(1) { #ifdef SOLARIS sigwait(&signal_set); #else sigwait(&signal_set, &signal); #endif switch(signal) { case SIGINT: sigint_handler(); break; case SIGALRM: sigalrm_handler(); break; } } } void sigint_handler(void) { int i; printf("^C caught, saving download job...\n"); for (i = 0; i < nthreads; i++) { pthread_cancel(wthread[i].tid); wthread[i].status &= STAT_INT; /* Interrupted download */ } save_log(); exit(0); } void sigalrm_handler(void) { printf("Signal Alarm came\n"); updateProgressBar(bwritten, req->clength); alarm(1); } aget-0.4.1/Aget.h0000644000076400000120000000021707566442007011755 0ustar mbwheel#ifndef AGET_H #define AGET_H #include "Data.h" #include "Resume.h" void get(struct request *); void resume_get(struct hist_data *); #endif aget-0.4.1/Head.h0000644000076400000120000000014007566442007011731 0ustar mbwheel#ifndef HEAD_H #define HEAD_H #include "Data.h" void http_head_req(struct request *); #endif aget-0.4.1/ChangeLog0000644000076400000120000000363211202204256012462 0ustar mbwheelEnderUNIX Aget ChangeLog * Tue May 12 08:25:29 EEST 2009 Version 0.4.1 adds/changes the following: - Fixed errno problem on newer glibc's - Large file support (files bigger than 2 GB) (Thanks to Siwei Liu of Sun Microsystems) * Fri Nov 22 07:44:20 EET 2002 Version 0.4 adds/changes the following: - Fixed bugs in Resume code. - Fixed miscalculations in the progress indicator * Fri Sep 20 18:11:11 EEST 2002 Version 0.39 adds/changes the following: - Hafiye is not 100 % POSIX compliant now. pread/pwrite functions are now used to simultaneously write to the local file. - Because of pwrite, the local file is not assembled at the end now, it's assembled at the same time. This speeds up large downloads; since previously lots of I/O was used to assemble the file. This is now decreased by at least fifty percent. - Solaris bug fixed. - Resume support. - Some more options: -p [port number to connect to] -l [local file name, if different from the remote file] -f [force use of suggested number of threads] - A progress indicator has been added. - Code has been re-arranged. Since aget starts to become larger, that was a need. - A README-Developer has been added for developers who want to contribute to aget. * Fri May 3 23:25:08 EEST 2002 Version 0.2p1 adds/changes the following: - getopt() problem is now solved, The program should really run on Posix systems. I've tested it on Linux, FreeBSD ans Solaris. - If no thread number is supplied, it is defaulted to 1. - A Solaris Makefile is added to the tarball. * Wed May 1 18:15:32 EEST 2002 Version 0.2 adds/changes the following: - File is now created seperately in each thread and later, it is assembled. Thus, some overhead from locking and unlocking of the mutex has been decreased. * Wed May 1 10:43:44 EEST 2002 First Public Release: 0.1 aget-0.4.1/aget.10000644000076400000120000000343611202734536011725 0ustar mbwheel.\" .TH "AGET" "1" "" "" "" .SH "NAME" aget \- A multithreaded HTTP download accelerator .SH "SYNOPSIS" .B aget .I "[options] file ..." .SH "DESCRIPTION" This manual page documents briefly the .BR aget commands. This manual page was originally written for the Debian GNU/Linux distribution. .PP .B aget fetches HTTP URLs in a manner similar to wget, but segments the retrieval into multiple parts to increase download speed. It can be many times as fast as wget in some circumstances. It is the starting point for a project attempting to replicate the functionality of a MS Windows program called Flashget. .SH "OPTIONS" .TP .B \-p port number .TP .B \-l local file name .TP .B \-h Show summary of options. .TP .B \-v Show version of program. .TP .B \-n Suggested number of threads .TP .B \-f force using suggested number of threads .SH EXIT STATUS .LP The following exit values are returned: .sp .ne 2 .mk .na \fB\fB0\fR \fR .ad .RS 7n .rt Successful completion. .RE .sp .ne 2 .mk .na \fB\fB1\fR \fR .ad .RS 7n .rt Application exited with failure. .RE .sp .ne 2 .mk .na \fB\fB-1\fR \fR .ad .RS 7n .rt POSIX pthreads creation error occurred. .RE .SH "EXAMPLES" .PP \fBExample 1: Say you want to download a \fIURL\fR using 10 \fIthreads\fR, just type:\fR .PP .PP .nf example% \fBaget -n 10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3\fR .fi .SH "AUTHOR" Aget tool is written by Murat Balaban, and project web page is: http://www.enderunix.org/aget/ .br This manual page was written by Bruno Barrera C. , for the Debian GNU/Linux system (but may be used by others). .SH "REPORTING BUGS" Report bugs to . .SH COPYRIGHT .br Please see the accompanying COPYRIGHT file that comes with the tarball. .SH BUGS .LP This program does not support IPv6 yet. aget-0.4.1/Misc.c0000644000076400000120000001106211202203657011747 0ustar mbwheel#include #include #include #include #include #include #include #include #include #include "Misc.h" #include "Data.h" void parse_url(char *url, struct request *req) { char *s; int i, j, k; i = j = k = 0; s = url; if ((strncmp(url, "ftp://", 6)) == 0) { fprintf(stderr, "Error: Currently Aget doesn't support FTP requests...\n"); exit(1); } else if ((strncmp(url, "http://", 7)) != 0) { fprintf(stderr, "Error: URL should be of the form http://...\n"); exit(1); } if (req->port == 0) { req->port = 80; req->proto = PROTO_HTTP; } s = url + 7; /* Jump pass http:// part */ for (i = 0; *s != '/'; i++, s++) { if (i > MAXHOSTSIZ) { fprintf(stderr, "Error: Cannot get hostname from URL...\n"); exit(1); } if (*s == ':') { /* If user/pass is supplied like; http://murat:12345@x.y.com/url.html */ while(*s != '/') { req->username[j++] = *--s; i--; } req->username[--j] = '\0'; revstr(req->username); while(1) { if (*s == ':') { while(*s != '@') { if (k > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get password from URL...\n"); exit(1); } req->password[k++] = *++s; } break; } s++; } req->password[--k] = '\0'; } req->host[i] = *s; } req->host[i] = '\0'; for (i = 0; *s != '\0'; i++, s++) { if (i > MAXURLSIZ) { fprintf(stderr, "Error: Cannot get remote file name from URL...\n"); exit(1); } req->url[i] = *s; } req->url[i] = '\0'; --s; for (i = 0; *s != '/'; i++, s--) { if (i > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get local file name from URL...\n"); exit(1); } req->file[i] = *s; } req->file[i] = '\0'; revstr(req->file); } int numofthreads(int size) { if (size == 0) return 0; else if (size < BUFSIZ * 2) /* < 16384 */ return 1; else if ((size >= BUFSIZ * 2) && (size < BUFSIZ * 4)) /* 16384 < x < 32678 */ return 2; else if ((size >= BUFSIZ * 4) && (size < BUFSIZ * 8)) /* 32768 < x < 65536 */ return 3; else if ((size >= BUFSIZ * 8) && (size < BUFSIZ * 16)) /* 65536 < x < 131072 */ return 4; else if ((size >= BUFSIZ * 16) && (size < BUFSIZ * 32)) /* 131072 < x < 262144 */ return 5; else if ((size >= BUFSIZ * 32) && (size < BUFSIZ * 64)) return 6; else if ((size >= BUFSIZ * 64) && (size < BUFSIZ * 128)) return 7; else if ((size >= BUFSIZ * 128) && (size < BUFSIZ * 256)) return 8; else if ((size >= BUFSIZ * 256) && (size < BUFSIZ * 512)) return 9; else return 10; } int calc_offset(long long total, int part, int nthreads) { return (part * (total / nthreads)); } void usage() { fprintf(stderr, "usage: aget [options] url\n"); fprintf(stderr, "\toptions:\n"); fprintf(stderr, "\t\t-p port number\n"); fprintf(stderr, "\t\t-l local file name\n"); fprintf(stderr, "\t\t-n suggested number of threads\n"); fprintf(stderr, "\t\t-f force using suggested number of threads\n"); fprintf(stderr, "\t\t-h this screen\n"); fprintf(stderr, "\t\t-v version info\n"); fprintf(stderr, "\n"); fprintf(stderr, "http//www.enderunix.org/aget/\n"); } /* reverse a given string */ void revstr(char *str) { char *p, *s; int i; int size; if ((size = strlen(str)) == 0) return; p = (char *)calloc(size, sizeof(char)); s = p; for (i = size; i >= 0; i--, s++) *s = *(str + i - 1); *s = '\0'; memset(str, 0, size); strncpy(str, p, size); free(p); } /* Log */ void Log(char *fmt, ...) { va_list ap; char *lfmt; lfmt = (char *)calloc(7 + strlen(fmt), sizeof(char)); sprintf(lfmt, " %s", fmt); fflush(stdout); va_start(ap, fmt); vfprintf(stderr, lfmt, ap); va_end(ap); if (fmt[0] != '\0' && fmt[strlen(fmt) - 1] == ':') fprintf(stderr, " %s", strerror(errno)); fprintf(stderr, "\n"); free(lfmt); } /* Progress Bar */ void updateProgressBar(float cur, float tot) { float rat; int ndot, i; static float prev = -1; rat = cur/tot; ndot = (int)(rat * 100); if ((ndot < prev + 5) && (ndot != 100)) return; for (i = 0; i < ndot; i += 2) putchar('.'); for (i = ndot - 1; i < 100; i += 2) putchar(' '); printf("[%d%% completed]\n", ndot); prev = ndot; } void handleHttpRetcode(char *rbuf) { if ((strstr(rbuf, "HTTP/1.1 416")) != NULL) { Log("Server returned HTTP/1.1 416 - Requested Range Not Satisfiable\n"); exit(1); } else if ((strstr(rbuf, "HTTP/1.1 403")) != NULL) { Log(" #include #include #include #include #include "Data.h" #include "Resume.h" #include "Misc.h" extern struct thread_data *wthread; extern struct request *req; extern int nthreads; extern int bwritten; void save_log() { char *logfile; struct hist_data h; FILE *fp; logfile = (char *)calloc(255, sizeof(char)); if (strlen(req[0].lfile) == 0) snprintf(logfile, 255, "aget-%s.log", req[0].file); else snprintf(logfile, 255, "aget-%s.log", req[0].lfile); if ((fp = fopen(logfile, "w")) == NULL) { fprintf(stderr, "cannot open log file %s for writing: %s\n", logfile, strerror(errno)); exit(1); } memcpy(&(h.req), req, sizeof(struct request)); memcpy(&(h.wthread), wthread, sizeof(struct thread_data) * nthreads); h.nthreads = nthreads; h.bwritten = bwritten; printf("--> Logfile is: %s, so far %d bytes have been transferred\n", logfile, h.bwritten); fwrite(&h, sizeof(struct hist_data), 1, fp); fclose(fp); free(logfile); } int read_log(struct hist_data *h) { char *logfile; FILE *fp; logfile = (char *)calloc(255, sizeof(char)); if (strlen(req[0].lfile) == 0) snprintf(logfile, 255, "aget-%s.log", req[0].file); else snprintf(logfile, 255, "aget-%s.log", req[0].lfile); Log("Attempting to read log file %s for resuming download job...", logfile); if ((fp = fopen(logfile, "r")) == NULL) { if (errno == ENOENT) { Log("Couldn't find log file for this download, starting a clean job..."); return -1; } else { fprintf(stderr, "cannot open log file %s for reading: %s\n", logfile, strerror(errno)); exit(1); } } fread(h, sizeof(struct hist_data), 1, fp); bwritten = h->bwritten; fclose(fp); Log("%d bytes already transferred", bwritten); /* Unlinking logfile after we've read it */ if ((unlink(logfile)) == -1) fprintf(stderr, "read_log: cannot remove stale log file %s: %s\n", logfile, strerror(errno)); free(logfile); return 0; } aget-0.4.1/Defs.h0000644000076400000120000000125611202206333011740 0ustar mbwheel#ifndef DEFS_H #define DEFS_H enum { GETREQSIZ = 256, GETRECVSIZ = 8192, HEADREQSIZ = 512, MAXURLSIZ = 1024, MAXHOSTSIZ = 1024, MAXIPSIZ = 16, MAXBUFSIZ = 512, MAXTHREADS = 25, HTTPPORT = 80, UNKNOWNREQ = 2, FTPREQ = 21, PROTO_HTTP = 0xFF, PROTO_FTP = 0x00, STAT_OK = 0xFF, /* Download completed successfully */ STAT_INT = 0x0F, /* ^C caught, download interrupted */ STAT_ERR = 0x00 /* Download finished with error */ }; #define PROGVERSION "EnderUNIX Aget v0.4.1" #define HEADREQ "HEAD %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n\r\n" #define GETREQ "GET %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nRange: bytes=%lld-\r\nConnection: close\r\n\r\n" #endif aget-0.4.1/TODO0000644000076400000120000000062411202206300011366 0ustar mbwheelEnderUNIX Aget v0.4.1 TODO: ------------------------- -> HTTP Proxy support -> Constructing libaget.so/libaget.a for IstanbulX project of acikkod.org -> Thread exit status evaluation: If one of the threads fail, the program should retry the failed later on. -> FreeBSD/NetBSD/OpenBSD Port/Package -> Multiple downloads -> FTP Support Tue May 12 08:44:31 EEST 2009 http://www.enderunix.org/aget/ aget-0.4.1/main.c0000644000076400000120000000465111202206003011772 0ustar mbwheel#include #include #include #include #include #include #include "Defs.h" #include "Data.h" #include "Misc.h" #include "Aget.h" #include "Signal.h" #include "Resume.h" #include "main.h" #include int main(int argc, char **argv) { extern char *optarg; extern int optind; int c, error = 0, ret; struct hist_data h; int retlog; /* Allocate heap for download request * struct request stores all the information that might be * of interest */ req = (struct request *)calloc(1, sizeof(struct request)); /* Only some signals will be emitted */ sigemptyset(&signal_set); sigaddset(&signal_set, SIGINT); sigaddset(&signal_set, SIGALRM); /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &signal_set, NULL); /* Create a thread for hadling signals */ if ((ret = pthread_create(&hthread, NULL, signal_waiter, NULL)) != 0) { fprintf(stderr, "main: cannot create signal_waiter thread: %s, exiting...\n", strerror(errno)); exit(-1); } while (!error && (c = getopt(argc,argv,"p:l:n:hfv")) != -1) { switch(c) { case 'p': req->port = atoi(optarg); break; case 'f': fsuggested = 1; break; case 'l': strncpy(req->lfile, optarg, MAXBUFSIZ); break; case 'n': if ((nthreads = atoi(optarg)) > MAXTHREADS) { Log("Error: Maximum # of threads allowed is %d\n", MAXTHREADS); nthreads = 0; } break; case 'h': printf("%s\n", PROGVERSION); usage(); exit(0); break; case 'v': printf("%s\nby Murat BALABAN \n", PROGVERSION); exit(0); break; default: error = 1; usage(); exit(1); break; } } if (error) { usage(); exit(1); } if (fsuggested == 1 && nthreads == 0) { fprintf(stderr, "\nERROR: -f and -n should be used together!, exiting...\n\n"); usage(); exit(1); } if (argc == 2) /* If only url is supplied... */ fullurl = strdup(argv[1]); else if (optind < argc) if (argc > 2) fullurl = strdup(argv[optind]); else { usage(); exit(1); } else if (optind == argc) { usage(); exit(1); } parse_url(fullurl, req); /* If a log file for a previous try has been found, read it and * resume the download job (resume_get), otherwise, start with * a clean job (get) * * Logfile is of the pattern: aget-$file_name.log */ if ((retlog = read_log(&h)) != -1) resume_get(&h); else get(req); return 0; } aget-0.4.1/COPYING0000644000076400000120000000347211202206172011744 0ustar mbwheel Copyright (c) 2002 , Murat Balaban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the EnderUNIX Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Tue May 12 08:43:20 EEST 2009 aget-0.4.1/Resume.h0000644000076400000120000000041107566714167012342 0ustar mbwheel#ifndef RESUME_H #define RESUME_H #include "Data.h" #include "Defs.h" typedef struct hist_data { struct request req; int nthreads; int bwritten; struct thread_data wthread[MAXTHREADS]; } hist_data; void save_log(); int read_log(struct hist_data *); #endif aget-0.4.1/Makefile.Solaris0000644000076400000120000000065511202454473013775 0ustar mbwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -W -DSOLARIS -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 LDFLAGS = -lpthread -lsocket -lnsl -lresolv CC = gcc STRIP=/usr/ccs/bin/strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ aget-0.4.1/THANKS0000644000076400000120000000041211202454565011626 0ustar mbwheel Sensei, and my EnderUNIX team members... Si-wei Louis Liu Atilim Boy nikinakof Oleg Mitsura Tue May 12 08:31:35 EEST 2009 aget-0.4.1/AUTHORS0000644000076400000120000000034411202204052011747 0ustar mbwheelCoder & Maintainer: ------------------- Murat Balaban For questions and comments, and also bug reports, don't hesitate to mail murat@enderunix.org http://www.enderunix.org/ Tue May 12 08:24:56 EEST 2009 aget-0.4.1/Download.h0000644000076400000120000000013607566442010012636 0ustar mbwheel#ifndef DOWNLOAD_H #define DOWNLOAD_H #include void * http_get(void *); #endif aget-0.4.1/README-Developer0000644000076400000120000000274311202206246013516 0ustar mbwheelOk, Here's a developer's primer for aget source: The code is splitted into several files: main.{c|h} * starts a thread for handling signals. * gets program options and decides the program's behviour by setting a few global variables. Aget.{c|h} * get() and resume_get() functions starts the download process. Depending on the protocol, necessary methods are called. resume_get() is for resume jobs. If the there is a log file of a previous job, the values are taken from it. Head.{c|h} * Before the actual download, a head request is sent to the server to determine the current status of the file. Some values like: 1. HTTP return code (e.g. 404, 206, 403, 200) 2. Content-length might be of interest there. Download.{c|h} * Aget is no more %100 Posix compliant :( pwrite is from X_OPEN standarts. I had to do it, since I needed a way to simultaneously write to the same file without loosing any time while synchronising the threads. For the time, there is only http_get. When aget is ftp-aware ftp_get will be in this file. Resume.{c|h} * Methods about saving and restoring history files are in this file. Signal.{c|h} * Signal Handling functions. Misc.{c|h} * Other functions which do not fit any of the general categories are stored here. These are generally helper functions... For more information read the aget source, and you may contact murat at enderunix dot org. Tue May 12 08:44:04 EEST 2009 aget-0.4.1/README0000644000076400000120000000075011202206232011562 0ustar mbwheelEnderUNIX Aget v0.4.1 README -------------------------- This program is a starting point for a very useful project like FlashGet for Win32. My aim is to provide all the functionality that program has. If you need assistance in installing Aget, you can see INSTALL file, Program is tested on RedHat Linux 8, FreeBSD 4.7 and Solaris 7,8 (Sparc). It proved to be successful. You can get much more info from the Aget Web site: http://www.enderunix.org/aget Tue May 12 08:43:52 EEST 2009 aget-0.4.1/INSTALL0000644000076400000120000000123611202206213011732 0ustar mbwheelEnderUNIX Aget v0.4.1 INSTALL -------------------------- The program is tested on RedHat Linux 7.2 (gcc-2.96), FreeBSD 4.5 and Solaris 8. For Solaris, you'll have to use Makefile.Solaris. To compile Aget, type: # make && make strip To install the compiled binary, type: # make install and binary will be installed as /usr/local/bin/aget You're done. All you need is a file to download and supply that url and desired number of segments to aget, e.g: $ aget -n10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3 Aget will split the file into 10 segments and will download it for you. Tue May 12 08:43:32 EEST 2009 http://www.enderunix.org/aget/