aget-0.2/ 40755 0 0 0 7464263066 10413 5ustar rootwheelaget-0.2/TODO100644 0 0 733 7464107574 11164 0ustar rootwheelEnderUNIX Aget v0.2 TODO: ------------------------- -> HTTP Proxy support -> Resume support -> Add some code to suggest thread number according to the size of the file. -> Thread exit status evaluation: If one of the threads fail, the program should retry the failed later on. -> FreeBSD/NetBSD/OpenBSD Port -> Solaris Port -> GUI -> Multiple downloads -> Active FTP support -> Passive FTP support Wed May 1 18:15:22 EEST 2002 http://www.enderunix.org/aget/ aget-0.2/misc.h100644 0 0 551 7464107574 11576 0ustar rootwheel /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #ifndef MISC_H #define MISC_H #include "socket.h" int calc_offset(int, int, int); void parse_url(char *, struct request *); void usage(); void revstr(char *); /* Reverse String */ #endif aget-0.2/misc.c100644 0 0 3166 7464107574 11616 0ustar rootwheel#include #include #include #include #include "misc.h" void parse_url(char *url, struct request *req) { char *s; int i; s = url; if ((strncmp(url, "ftp://", 6)) == 0) { fprintf(stderr, "Error: Currently Aget doesn't support FTP requests...\n"); exit(1); } else if ((strncmp(url, "http://", 7)) != 0) { fprintf(stderr, "Error: URL should be of the form http://...\n"); exit(1); } req->port = 80; s = url + 7; /* Jump pass http:// part */ for (i = 0; *s != '/'; i++, s++) { if (i > MAXHOSTSIZ) { fprintf(stderr, "Error: Cannot get hostname from URL...\n"); exit(1); } req->host[i] = *s; } req->host[i] = '\0'; for (i = 0; *s != '\0'; i++, s++) { if (i > MAXURLSIZ) { fprintf(stderr, "Error: Cannot get remote file name from URL...\n"); exit(1); } req->url[i] = *s; } req->url[i] = '\0'; --s; for (i = 0; *s != '/'; i++, s--) { if (i > MAXFILESIZ) { fprintf(stderr, "Error: Cannot get local file name from URL...\n"); exit(1); } req->file[i] = *s; } req->file[i] = '\0'; revstr(req->file); } int calc_offset(int total, int part, int nthreads) { return (part * (total / nthreads)); } void usage() { fprintf(stderr, "\n%s [http://www.enderunix.org/aget/]\n\n", PROGVERSION); fprintf(stderr, "usage: aget url -n [num of threads]\n"); } /* reverse a given string */ void revstr(char *str) { char *p, *s; int i; int size; if ((size = strlen(str)) == 0) return; p = (char *)calloc(size, sizeof(char)); s = p; for (i = size; i >= 0; i--, s++) *s = *(str + i - 1); *s = '\0'; memset(str, 0, size); strncpy(str, p, size); free(p); } aget-0.2/socket.c100644 0 0 12063 7464107574 12167 0ustar rootwheel /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #include #include #include #include #include #include #include #include #include #include "socket.h" extern int errno; void head_req(struct request *req) { struct sockaddr_in sin; struct hostent *he; int sd; char *sbuf; char *rbuf; char *tok; char *s; int clength; sbuf = (char *)calloc(HEADREQSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(HEADRECVSIZ, sizeof(char)); if ((he = gethostbyname(req->host)) == NULL) { fprintf(stderr, "Error: Cannot resolve hostname\n"); herror("gethostbyname"); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); sin.sin_port = htons(req->port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("socket"); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { perror("connect"); exit(1); } Log("Head-Request Connection established"); sprintf(sbuf, HEADREQ, req->url, req->host, PROGVERSION); if ((send(sd, sbuf, strlen(sbuf), 0)) == -1) { perror("send"); exit(1); } if ((recv(sd, rbuf, HEADRECVSIZ, 0)) == -1) { perror("recv"); exit(1); } if ((strstr((tok = strtok(rbuf, "\r\n")), "HTTP/1.1 404")) != NULL) { printf(" File Not Found!, Aborting...\n"); close(sd); exit(1); } else if ((strstr(tok, "HTTP/1.1 200")) != NULL) { while ((tok = strtok(NULL, "\r\n")) != NULL) { if ((strstr(tok, "Content-Length")) != NULL) { s = (tok + strlen("Content-Length: ")); clength = atoi(s); printf(" Content-Length: %d\n", clength); req->clength = clength; } } } free(sbuf); free(rbuf); } void *partialGetHandler(void *arg) { thread_data *td; struct sockaddr_in sin; int sd; char *req, *rbuf, *s; int offset, dr, dw, i; long foffset; pthread_t tid; FILE *fp; char *file; tid = pthread_self(); printf(" Entering thread ...\n", tid); td = (thread_data *)arg; file = (char *)calloc(strlen(td->file) + 10, sizeof(char)); sprintf(file, "%s.tmp%d", td->file, td->tind); if ((fp = fopen(file, "w")) == NULL) { fprintf(stderr, " cannot create temporary file %s: %s\n", tid, file, strerror(errno)); exit(1); } foffset = td->foffset; printf(" Range: %ld-%ld\n", tid, td->soffset, td->foffset); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr = inet_addr(td->ip); sin.sin_port = htons(td->port); req = (char *)calloc(GETREQSIZ, sizeof(char)); rbuf = (char *)calloc(GETRECVSIZ, sizeof(char)); sprintf(req, GETREQ, td->url, td->host, PROGVERSION, td->soffset); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("thread socket"); free(td); free(req); pthread_exit((void *)1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { perror("thread connect"); free(td); free(req); pthread_exit((void *)1); } if ((send(sd, req, strlen(req), 0)) == -1) { perror("send"); free(td); free(req); pthread_exit((void *)1); } if ((dr = recv(sd, rbuf, HEADRECVSIZ, 0)) == -1) { perror("recv"); free(td); free(req); pthread_exit((void *)1); } if ((strstr(rbuf, "HTTP/1.1 404")) != NULL) { printf(" Server returned HTTP/1.1 404 - File Not Found, aborting\n", tid); free(td); free(req); close(sd); exit(1); } else if ((strstr(rbuf, "HTTP/1.1 206")) != NULL) printf(" HTTP/1.1 206 - Partial Content\n", tid); s = rbuf; i = 0; while(1) { if (*s == '\n' && *(s - 1) == '\r' && *(s - 2) == '\n' && *(s - 3) == '\r') { s++; i++; break; } s++; i++; } if ((dr - i ) > foffset) dw = fwrite(s, sizeof(char), (foffset - i), fp); else dw = fwrite(s, sizeof(char), (dr - i), fp); offset = td->soffset + dw; while (offset < foffset) { memset(rbuf, GETRECVSIZ, 0); dr = recv(sd, rbuf, GETRECVSIZ, 0); if ((offset + dr) > foffset) dw = fwrite(rbuf, sizeof(char), foffset - offset, fp); else dw = fwrite(rbuf, sizeof(char), dr, fp); offset += dw; /*printf(" offset: %d bytes, foffset: %d\n", tid, offset, foffset);*/ } free(req); free(td); close(sd); fclose(fp); printf(" Leaving thread...\n", tid); pthread_mutex_lock(&thread_exit_mtx); thread_exit_cnt--; pthread_cond_signal(&thread_exit_cnd); pthread_mutex_unlock(&thread_exit_mtx); pthread_exit(NULL); return NULL; } void Log(char *log) { #ifdef DEBUG fprintf(stderr, " %s\n", log); #endif } aget-0.2/socket.h100644 0 0 2523 7464107574 12154 0ustar rootwheel /* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #ifndef S_H #define S_H #include #include #define HEADREQSIZ 512 #define HEADRECVSIZ 2048 #define GETREQSIZ 2048 #define GETRECVSIZ 2048 #define HTTPPORT 80 #define FTPREQ 21 #define UNKNOWNREQ 2 #define MAXURLSIZ 1024 #define MAXHOSTSIZ 256 #define MAXFILESIZ 512 #define MAXIPSIZ 16 #define PROGVERSION "EnderUNIX Aget v0.2" #define HEADREQ "HEAD %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n\r\n" #define GETREQ "GET %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nRange: bytes=%ld-\r\nConnection: close\r\n\r\n" static pthread_mutex_t thread_exit_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t thread_exit_cnd = PTHREAD_COND_INITIALIZER; static int thread_exit_cnt = 0; typedef struct request { char host[MAXHOSTSIZ]; char url[MAXURLSIZ]; char file[MAXFILESIZ]; char ip[MAXIPSIZ]; int port; int clength; } request; typedef struct thread_data { char host[MAXHOSTSIZ]; char url[MAXURLSIZ]; char ip[MAXIPSIZ]; char file[MAXFILESIZ]; int port; long soffset; long foffset; FILE *fp; int tind; } thread_data; void Log(char *); void head_req(struct request *); void *partialGetHandler(void *); #endif aget-0.2/main.c100644 0 0 5631 7464107574 11606 0ustar rootwheel/* * * Multithreaded HTTP Download Accelarator: Aget * (c) 2002, Murat Balaban * * See COPYING for copyright and copying restrictions * */ #include #include #include #include #include #include #include #include #include #include #include "socket.h" #include "misc.h" #define HTTPPORT 80 #define FTPREQ 21 #define UNKNOWNREQ 2 #define MAXTHREADS 10 int main(int argc, char **argv) { struct request req; pthread_t *threads[MAXTHREADS]; pthread_t *tid; thread_data *tdata; int i, c, error = 0, r; int nthreads = 0; int soffset, foffset; FILE *fp, *tmpfp; extern char *optarg; extern int optind; char *rbuf, *tmpfile; rbuf = (char *)calloc(BUFSIZ, sizeof(char)); tmpfile = (char *)calloc(MAXFILESIZ, sizeof(char)); if (argc < 3) { usage(); exit(1); } parse_url(argv[1], &req); while (!error && (c = getopt(argc,argv,"n:hv")) != -1) { switch(c) { case 'n': nthreads = atoi(optarg); break; case 'h': printf("%s\n", PROGVERSION); usage(); exit(0); break; default: error = 1; usage(); exit(1); break; } } if (error) { usage(); exit(1); } head_req(&req); printf(" Downloading %s from site %s(%s:%d). Number of threads: %d\n", req.url, req.host, req.ip, req.port, nthreads); printf(" Local File: %s\n", req.file); for (i = 0; i < nthreads; i++) { soffset = calc_offset(req.clength, i, nthreads); foffset = calc_offset(req.clength, i + 1, nthreads); tid = (pthread_t *)malloc(sizeof(pthread_t)); tdata = (thread_data *)malloc(sizeof(thread_data)); strncpy(tdata->host, req.host, MAXHOSTSIZ); strncpy(tdata->url, req.url, MAXURLSIZ); strncpy(tdata->ip, req.ip, MAXIPSIZ); strncpy(tdata->file, req.file, MAXFILESIZ); tdata->soffset = soffset; tdata->foffset = (i == nthreads - 1 ? req.clength : foffset); tdata->port = req.port; tdata->fp = fp; tdata->tind = i; thread_exit_cnt++; pthread_create(tid, NULL, partialGetHandler, tdata); threads[i] = tid; } threads[i] = NULL; /* Later ! pthread_mutex_lock(&thread_exit_mtx); while (thread_exit_cnt != 0) pthread_cond_wait(&thread_exit_cnd, &thread_exit_mtx); pthread_mutex_unlock(&thread_exit_mtx); */ for (i = 0; threads[i] != NULL; i++) pthread_join(*threads[i], NULL); Log("Download complete, assembling file..."); if ((fp = fopen(req.file, "w")) == NULL) { perror("fopen"); exit(1); } for (i = 0; i < nthreads; i++) { sprintf(tmpfile, "%s.tmp%d", req.file, i); if ((tmpfp = fopen(tmpfile, "r")) == NULL) { perror("fopen"); exit(1); } while ((r = fread(rbuf, sizeof(char), BUFSIZ, tmpfp)) > 0) fwrite(rbuf, sizeof(char), r, fp); fclose(tmpfp); unlink(tmpfile); } Log("File assembly finished, job completed successfully!"); Log("Shutting down..."); fclose(fp); return 0; } aget-0.2/COPYING100444 0 0 3462 7464107574 11547 0ustar rootwheel Copyright (c) 2002 , Murat Balaban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the EnderUNIX Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Wed May 1 18:15:37 EEST 2002 aget-0.2/AUTHORS100644 0 0 344 7464107574 11542 0ustar rootwheelCoder & Maintainer: ------------------- Murat Balaban For questions and comments, and also bug reports, don't hesitate to mail murat@enderunix.org http://www.enderunix.org/ Wed May 1 18:15:15 EEST 2002 aget-0.2/ChangeLog100644 0 0 515 7464263063 12240 0ustar rootwheelEnderUNIX Aget ChangeLog * Wed May 1 18:15:32 EEST 2002 Version 0.2 adds/changes the following: - File is now created seperately in each thread and later, it is assembled. Thus, some overhead from locking and unlocking of the mutex has been decreased. * Wed May 1 10:43:44 EEST 2002 First Public Release: 0.1 aget-0.2/INSTALL100644 0 0 1077 7464107574 11547 0ustar rootwheelEnderUNIX Aget v0.2 INSTALL -------------------------- The program is tested on RedHat Linux 7.2 (gcc-2.96) To compile Aget, type: # make To install the compiled binary, type: # make install and binary will be installed as /usr/local/bin/aget You're done. All you need is a file to download and supply that url and desired number of segments to aget, e.g: $ aget http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3 -n10 Aget will split the file into 10 segments and will download it for you. Wed May 1 18:15:17 EEST 2002 http://www.enderunix.org/aget/ aget-0.2/README100644 0 0 1516 7464107574 11374 0ustar rootwheelEnderUNIX Aget v0.2 README -------------------------- This program is a starting point for a very useful project like FlashGet for Win32. My aim is to provide all the functionality that program has. This release is the first step. You can see TODO file for further planned development for Aget. If you need assistance in installing Aget, you can see INSTALL file, Program is tested on RedHat Linux 7.2 (gcc-2.96) and proved to be successful. A file of size 4,482,334 bytes was transferred in 1m2.159s with Aget, whereas wget downloaded the same file from the same location in 3m18.746s. The program is not tested on other platforms, however I expect to port it to BSD platforms in a few days... You can get much more info from the Aget Web site: http://www.enderunix.org/aget Wed May 1 18:15:19 EEST 2002 http://www.enderunix.org/aget/ aget-0.2/THANKS100644 0 0 102 7464107574 11375 0ustar rootwheel Sensei, and my EnderUNIX team members... aget-0.2/Makefile100644 0 0 433 7464107574 12131 0ustar rootwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = socket.o main.o misc.o CFLAGS = -Wall -W -pedantic -DDEBUG=y LDFLAGS = -lpthread CC = gcc all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~