aget-0.2p1/ 40755 0 24 0 7464625674 10710 5ustar rootstaffaget-0.2p1/Makefile100644 0 24 435 7464574030 12414 0ustar rootstaff# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = socket.o main.o misc.o CFLAGS = -Wall -W -pedantic -DDEBUG=y -g LDFLAGS = -pthread CC = gcc all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ aget-0.2p1/misc.h100644 0 24 551 7464107574 12064 0ustar rootstaff /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #ifndef MISC_H #define MISC_H #include "socket.h" int calc_offset(int, int, int); void parse_url(char *, struct request *); void usage(); void revstr(char *); /* Reverse String */ #endif aget-0.2p1/misc.c100644 0 24 3213 7464573516 12100 0ustar rootstaff#include #include #include #include #include #include "misc.h" void parse_url(char *url, struct request *req) { char *s; int i; s = url; if ((strncmp(url, "ftp://", 6)) == 0) { fprintf(stderr, "Error: Currently Aget doesn't support FTP requests...\n"); exit(1); } else if ((strncmp(url, "http://", 7)) != 0) { fprintf(stderr, "Error: URL should be of the form http://...\n"); exit(1); } req->port = 80; s = url + 7; /* Jump pass http:// part */ for (i = 0; *s != '/'; i++, s++) { if (i > MAXHOSTSIZ) { fprintf(stderr, "Error: Cannot get hostname from URL...\n"); exit(1); } req->host[i] = *s; } req->host[i] = '\0'; for (i = 0; *s != '\0'; i++, s++) { if (i > MAXURLSIZ) { fprintf(stderr, "Error: Cannot get remote file name from URL...\n"); exit(1); } req->url[i] = *s; } req->url[i] = '\0'; --s; for (i = 0; *s != '/'; i++, s--) { if (i > MAXFILESIZ) { fprintf(stderr, "Error: Cannot get local file name from URL...\n"); exit(1); } req->file[i] = *s; } req->file[i] = '\0'; revstr(req->file); } int calc_offset(int total, int part, int nthreads) { return (part * (total / nthreads)); } void usage() { fprintf(stderr, "\n%s [http://www.enderunix.org/aget/]\n\n", PROGVERSION); fprintf(stderr, "usage: aget -n [num of threads] url\n"); } /* reverse a given string */ void revstr(char *str) { char *p, *s; int i; int size; if ((size = strlen(str)) == 0) return; p = (char *)calloc(size, sizeof(char)); s = p; for (i = size; i >= 0; i--, s++) *s = *(str + i - 1); *s = '\0'; memset(str, 0, size); strncpy(str, p, size); free(p); } aget-0.2p1/socket.c100644 0 24 12110 7464573505 12447 0ustar rootstaff /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #include #include #include #include #include #include #include #include #include #include #include "socket.h" extern int errno; void head_req(struct request *req) { struct sockaddr_in sin; struct hostent *he; int sd; char *sbuf; char *rbuf; char *tok; char *s; int clength; sbuf = (char *)calloc(HEADREQSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(HEADRECVSIZ, sizeof(char)); if ((he = gethostbyname(req->host)) == NULL) { fprintf(stderr, "Error: Cannot resolve hostname\n"); herror("gethostbyname"); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); sin.sin_port = htons(req->port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("socket"); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { perror("connect"); exit(1); } Log("Head-Request Connection established"); sprintf(sbuf, HEADREQ, req->url, req->host, PROGVERSION); if ((send(sd, sbuf, strlen(sbuf), 0)) == -1) { perror("send"); exit(1); } if ((recv(sd, rbuf, HEADRECVSIZ, 0)) == -1) { perror("recv"); exit(1); } if ((strstr((tok = strtok(rbuf, "\r\n")), "HTTP/1.1 404")) != NULL) { printf(" File Not Found!, Aborting...\n"); close(sd); exit(1); } else if ((strstr(tok, "HTTP/1.1 200")) != NULL) { while ((tok = strtok(NULL, "\r\n")) != NULL) { if ((strstr(tok, "Content-Length")) != NULL) { s = (tok + strlen("Content-Length: ")); clength = atoi(s); printf(" Content-Length: %d\n", clength); req->clength = clength; } } } free(sbuf); free(rbuf); } void *partialGetHandler(void *arg) { thread_data *td; struct sockaddr_in sin; int sd; char *req, *rbuf, *s; int offset, dr, dw, i; long foffset; pthread_t tid; FILE *fp; char *file; tid = pthread_self(); printf(" Entering thread ...\n", tid); td = (thread_data *)arg; file = (char *)calloc(strlen(td->file) + 10, sizeof(char)); sprintf(file, "%s.tmp%d", td->file, td->tind); if ((fp = fopen(file, "w")) == NULL) { fprintf(stderr, " cannot create temporary file %s: %s\n", tid, file, strerror(errno)); exit(1); } foffset = td->foffset; printf(" Range: %ld-%ld\n", tid, td->soffset, td->foffset); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr = inet_addr(td->ip); sin.sin_port = htons(td->port); req = (char *)calloc(GETREQSIZ, sizeof(char)); rbuf = (char *)calloc(GETRECVSIZ, sizeof(char)); sprintf(req, GETREQ, td->url, td->host, PROGVERSION, td->soffset); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("thread socket"); free(td); free(req); pthread_exit((void *)1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { perror("thread connect"); free(td); free(req); pthread_exit((void *)1); } if ((send(sd, req, strlen(req), 0)) == -1) { perror("send"); free(td); free(req); pthread_exit((void *)1); } if ((dr = recv(sd, rbuf, HEADRECVSIZ, 0)) == -1) { perror("recv"); free(td); free(req); pthread_exit((void *)1); } if ((strstr(rbuf, "HTTP/1.1 404")) != NULL) { printf(" Server returned HTTP/1.1 404 - File Not Found, aborting\n", tid); free(td); free(req); close(sd); exit(1); } else if ((strstr(rbuf, "HTTP/1.1 206")) != NULL) printf(" HTTP/1.1 206 - Partial Content\n", tid); s = rbuf; i = 0; while(1) { if (*s == '\n' && *(s - 1) == '\r' && *(s - 2) == '\n' && *(s - 3) == '\r') { s++; i++; break; } s++; i++; } if ((dr - i ) > foffset) dw = fwrite(s, sizeof(char), (foffset - i), fp); else dw = fwrite(s, sizeof(char), (dr - i), fp); offset = td->soffset + dw; while (offset < foffset) { memset(rbuf, GETRECVSIZ, 0); dr = recv(sd, rbuf, GETRECVSIZ, 0); if ((offset + dr) > foffset) dw = fwrite(rbuf, sizeof(char), foffset - offset, fp); else dw = fwrite(rbuf, sizeof(char), dr, fp); offset += dw; /*printf(" offset: %d bytes, foffset: %d\n", tid, offset, foffset);*/ } free(req); free(td); close(sd); fclose(fp); printf(" Leaving thread...\n", tid); pthread_mutex_lock(&thread_exit_mtx); thread_exit_cnt--; pthread_cond_signal(&thread_exit_cnd); pthread_mutex_unlock(&thread_exit_mtx); pthread_exit(NULL); return NULL; } void Log(char *log) { #ifdef DEBUG fprintf(stderr, " %s\n", log); #endif } aget-0.2p1/socket.h100644 0 24 2523 7464107574 12442 0ustar rootstaff /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #ifndef S_H #define S_H #include #include #define HEADREQSIZ 512 #define HEADRECVSIZ 2048 #define GETREQSIZ 2048 #define GETRECVSIZ 2048 #define HTTPPORT 80 #define FTPREQ 21 #define UNKNOWNREQ 2 #define MAXURLSIZ 1024 #define MAXHOSTSIZ 256 #define MAXFILESIZ 512 #define MAXIPSIZ 16 #define PROGVERSION "EnderUNIX Aget v0.2" #define HEADREQ "HEAD %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n\r\n" #define GETREQ "GET %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nRange: bytes=%ld-\r\nConnection: close\r\n\r\n" static pthread_mutex_t thread_exit_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t thread_exit_cnd = PTHREAD_COND_INITIALIZER; static int thread_exit_cnt = 0; typedef struct request { char host[MAXHOSTSIZ]; char url[MAXURLSIZ]; char file[MAXFILESIZ]; char ip[MAXIPSIZ]; int port; int clength; } request; typedef struct thread_data { char host[MAXHOSTSIZ]; char url[MAXURLSIZ]; char ip[MAXIPSIZ]; char file[MAXFILESIZ]; int port; long soffset; long foffset; FILE *fp; int tind; } thread_data; void Log(char *); void head_req(struct request *); void *partialGetHandler(void *); #endif aget-0.2p1/main.c100644 0 24 6450 7464573473 12101 0ustar rootstaff/* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #include #include #include #include #include #include #include #include #include #include #include #include "socket.h" #include "misc.h" #define HTTPPORT 80 #define FTPREQ 21 #define UNKNOWNREQ 2 #define MAXTHREADS 10 int main(int argc, char **argv) { struct request req; pthread_t *threads[MAXTHREADS]; pthread_t *tid; thread_data *tdata; int i, c, error = 0, r; int nthreads = 0; int soffset, foffset; FILE *fp, *tmpfp; extern char *optarg; extern int optind; char *rbuf, *tmpfile; char *fullurl; rbuf = (char *)calloc(BUFSIZ, sizeof(char)); tmpfile = (char *)calloc(MAXFILESIZ, sizeof(char)); while (!error && (c = getopt(argc,argv,"n:hv")) != -1) { switch(c) { case 'n': nthreads = atoi(optarg); break; case 'h': printf("%s\n", PROGVERSION); usage(); exit(0); break; case 'v': puts(PROGVERSION); exit(0); break; default: error = 1; usage(); exit(1); break; } } if (error) { usage(); exit(1); } if (argc == 2) { /* If only url is supplied... */ fullurl = strdup(argv[1]); printf("Warning: number of threads is not specified, defaulting to 1\n"); nthreads = 1; } else if (optind < argc) if (argc > 2) fullurl = strdup(argv[optind]); else { usage(); exit(1); } else if (optind == argc) { usage(); exit(1); } parse_url(fullurl, &req); head_req(&req); printf(" Downloading %s from site %s(%s:%d). Number of threads: %d\n", req.url, req.host, req.ip, req.port, nthreads); printf(" Local File: %s\n", req.file); for (i = 0; i < nthreads; i++) { soffset = calc_offset(req.clength, i, nthreads); foffset = calc_offset(req.clength, i + 1, nthreads); tid = (pthread_t *)malloc(sizeof(pthread_t)); tdata = (thread_data *)malloc(sizeof(thread_data)); strncpy(tdata->host, req.host, MAXHOSTSIZ); strncpy(tdata->url, req.url, MAXURLSIZ); strncpy(tdata->ip, req.ip, MAXIPSIZ); strncpy(tdata->file, req.file, MAXFILESIZ); tdata->soffset = soffset; tdata->foffset = (i == nthreads - 1 ? req.clength : foffset); tdata->port = req.port; tdata->fp = fp; tdata->tind = i; thread_exit_cnt++; pthread_create(tid, NULL, partialGetHandler, tdata); threads[i] = tid; } threads[i] = NULL; /* Later ! pthread_mutex_lock(&thread_exit_mtx); while (thread_exit_cnt != 0) pthread_cond_wait(&thread_exit_cnd, &thread_exit_mtx); pthread_mutex_unlock(&thread_exit_mtx); */ for (i = 0; threads[i] != NULL; i++) pthread_join(*threads[i], NULL); Log("Download complete, assembling file..."); if ((fp = fopen(req.file, "w")) == NULL) { perror("fopen"); exit(1); } for (i = 0; i < nthreads; i++) { sprintf(tmpfile, "%s.tmp%d", req.file, i); if ((tmpfp = fopen(tmpfile, "r")) == NULL) { perror("fopen"); exit(1); } while ((r = fread(rbuf, sizeof(char), BUFSIZ, tmpfp)) > 0) fwrite(rbuf, sizeof(char), r, fp); fclose(tmpfp); unlink(tmpfile); } Log("File assembly finished, job completed successfully!"); Log("Shutting down..."); fclose(fp); return 0; } aget-0.2p1/COPYING100444 0 24 3462 7464571505 12034 0ustar rootstaff Copyright (c) 2002 , Murat Balaban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the EnderUNIX Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Fri May 3 23:29:27 EEST 2002 aget-0.2p1/AUTHORS100644 0 24 344 7464571466 12035 0ustar rootstaffCoder & Maintainer: ------------------- Murat Balaban For questions and comments, and also bug reports, don't hesitate to mail murat@enderunix.org http://www.enderunix.org/ Fri May 3 23:29:17 EEST 2002 aget-0.2p1/ChangeLog100644 0 24 1221 7464572226 12545 0ustar rootstaffEnderUNIX Aget ChangeLog * Fri May 3 23:25:08 EEST 2002 Version 0.2p1 adds/changes the following: - getopt() problem is now solved, The program should really run on Posix systems. I've tested it on Linux, FreeBSD ans Solaris. - If no thread number is supplied, it is defaulted to 1. - A Solaris Makefile is added to the tarball. * Wed May 1 18:15:32 EEST 2002 Version 0.2 adds/changes the following: - File is now created seperately in each thread and later, it is assembled. Thus, some overhead from locking and unlocking of the mutex has been decreased. * Wed May 1 10:43:44 EEST 2002 First Public Release: 0.1 aget-0.2p1/INSTALL100644 0 24 1216 7464625554 12033 0ustar rootstaffEnderUNIX Aget v0.2 INSTALL -------------------------- The program is tested on RedHat Linux 7.2 (gcc-2.96), FreeBSD 4.5 and Solaris 8. For Solaris, you'll have to use Makefile.Solaris. To compile Aget, type: # make To install the compiled binary, type: # make install and binary will be installed as /usr/local/bin/aget You're done. All you need is a file to download and supply that url and desired number of segments to aget, e.g: $ aget -n10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3 Aget will split the file into 10 segments and will download it for you. Fri May 3 23:29:20 EEST 2002 http://www.enderunix.org/aget/ aget-0.2p1/README100644 0 24 1404 7464625643 11660 0ustar rootstaffEnderUNIX Aget v0.2 README -------------------------- This program is a starting point for a very useful project like FlashGet for Win32. My aim is to provide all the functionality that program has. This release is the first step. You can see TODO file for further planned development for Aget. If you need assistance in installing Aget, you can see INSTALL file, Program is tested on RedHat Linux 7.2 (gcc-2.96), FreeBSD 4.5 and Solaris 8 (Sparc). It proved to be successful. A file of size 4,482,334 bytes was transferred in 1m2.159s with Aget, whereas wget downloaded the same file from the same location in 3m18.746s. You can get much more info from the Aget Web site: http://www.enderunix.org/aget Fri May 3 23:30:13 EEST 2002 http://www.enderunix.org/aget/ aget-0.2p1/THANKS100644 0 24 177 7464571562 11701 0ustar rootstaff Sensei, and my EnderUNIX team members... Atilim Boy Fri May 3 23:29:30 EEST 2002 aget-0.2p1/TODO100644 0 24 743 7464625663 11457 0ustar rootstaffEnderUNIX Aget v0.2 TODO: ------------------------- -> HTTP Proxy support -> Resume support -> Add some code to suggest thread number according to the size of the file. -> Thread exit status evaluation: If one of the threads fail, the program should retry the failed later on. -> FreeBSD/NetBSD/OpenBSD Port/Package -> Solaris Port -> GUI -> Multiple downloads -> Active FTP support -> Passive FTP support Fri May 3 23:29:35 EEST 2002 http://www.enderunix.org/aget/ aget-0.2p1/Makefile.Solaris100644 0 24 466 7464574217 14042 0ustar rootstaff# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = socket.o main.o misc.o CFLAGS = -Wall -W -pedantic -DDEBUG=y -g LDFLAGS = -lpthread -lsocket -lnsl -lresolv CC = gcc all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ aget-0.2p1/Makefile.orig100644 0 24 435 7464571043 13354 0ustar rootstaff# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = socket.o main.o misc.o CFLAGS = -Wall -W -pedantic -DDEBUG=y -g LDFLAGS = -pthread CC = gcc all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~