aget-0.4/0040755000175000000000000000000007667330414011314 5ustar muratwheelaget-0.4/AUTHORS0100644000175000000000000000034307567343561012367 0ustar muratwheelCoder & Maintainer: ------------------- Murat Balaban For questions and comments, and also bug reports, don't hesitate to mail murat@enderunix.org http://www.enderunix.org/ Fri Nov 22 07:43:40 EET 2002 aget-0.4/Aget.c0100644000175000000000000001242607567363157012353 0ustar muratwheel #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Aget.h" #include "Misc.h" #include "Download.h" #include "Resume.h" #include "Data.h" extern struct thread_data *wthread; extern struct request *req; extern int fsuggested, nthreads; extern int bwritten; extern pthread_t hthread; extern int errno; void get(struct request *req) { int i, ret, fd, diff_sec, nok = 0; long soffset, foffset; char *fmt; if (req->proto == PROTO_HTTP) http_head_req(req); /* According to the content-length, get the * suggested number of threads to open. * if the user insists on his value, let it be that way, * use the user's value. */ ret = numofthreads(req->clength); if (fsuggested == 0) { if (ret == 0) nthreads = 1; else nthreads = ret; } wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data)); Log("Downloading %s (%d bytes) from site %s(%s:%d). Number of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if (strlen(req->lfile) != 0) { if ((fd = open(req->lfile, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else { if ((fd = open(req->file, O_CREAT | O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } if ((lseek(fd, req->clength - 1, SEEK_SET)) == -1) { fprintf(stderr, "get: couldn't lseek: %s\n", strerror(errno)); exit(1); } if ((write(fd, "0", 1)) == -1) { fprintf(stderr, "get: couldn't allocate space for download file: %s\n", strerror(errno)); exit(1); } /* Get the starting time, prepare GET format string, and start the threads */ fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); time(&t_start); for (i = 0; i < nthreads; i++) { soffset = calc_offset(req->clength, i, nthreads); foffset = calc_offset(req->clength, i + 1, nthreads); wthread[i].soffset = soffset; wthread[i].foffset = (i == nthreads - 1 ? req->clength : foffset); wthread[i].sin.sin_family = AF_INET; wthread[i].sin.sin_addr.s_addr = inet_addr(req->ip); wthread[i].sin.sin_port = htons(req->port); wthread[i].fd = dup(fd); wthread[i].clength = req->clength; snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, soffset); strncpy(wthread[i].getstr, fmt, GETREQSIZ); pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])); } free(fmt); /* Wait for all of the threads to finish... * * TODO: If a thread fails, restart that! */ for (i = 0; i < nthreads; i++) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } if (nok == nthreads) pthread_cancel(hthread); else pthread_join(hthread, NULL); /* Get the finish time, derive some stats */ time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%d Kb/sec)", diff_sec, (req->clength / diff_sec) / 1024); Log("Shutting down..."); close(fd); } void resume_get(struct hist_data *h) { int i, fd, diff_sec, nok = 0; char *fmt; nthreads = h->nthreads; fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char)); wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data)); memcpy(req, &h->req, sizeof(struct request)); memcpy(wthread, h->wthread, sizeof(struct thread_data) * nthreads); Log("Resuming download %s (%d bytes) from site %s(%s:%d). Number of Threads: %d", req->url, req->clength, req->host, req->ip, req->port, nthreads); if (strlen(req->lfile) != 0) { if ((fd = open(req->lfile, O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } else { if ((fd = open(req->file, O_RDWR, S_IRWXU)) == -1) { fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno)); exit(1); } } time(&t_start); #ifdef DEBUG for (i = 0; i < nthreads; i++) printf("Start: %ld, Finish: %ld, Offset: %ld, Diff: %ld\n", wthread[i].soffset, wthread[i].foffset, wthread[i].offset, wthread[i].offset - wthread[i].soffset); #endif for (i = 0; i < nthreads; i++) { wthread[i].soffset = wthread[i].offset; wthread[i].fd = dup(fd); snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, wthread[i].offset); strncpy(wthread[i].getstr, fmt, GETREQSIZ); pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i])); } for (i = 0; i < nthreads; i++) pthread_join(wthread[i].tid, NULL); for (i = 0; i < nthreads; i++) { pthread_join(wthread[i].tid, NULL); if (wthread[i].status == STAT_OK) nok++; } if (nok == nthreads) pthread_cancel(hthread); else pthread_join(hthread, NULL); time(&t_finish); if ((diff_sec = t_finish - t_start) == 0) diff_sec = 1; /* Avoid division by zero */ Log("Download completed, job completed in %d seconds. (%d Kb/sec)", diff_sec, ((req->clength - h->bwritten) / diff_sec) / 1024); Log("Shutting down..."); close(fd); } aget-0.4/Aget.h0100644000175000000000000000021707566442007012342 0ustar muratwheel#ifndef AGET_H #define AGET_H #include "Data.h" #include "Resume.h" void get(struct request *); void resume_get(struct hist_data *); #endif aget-0.4/COPYING0100644000175000000000000000347007567343561012356 0ustar muratwheel Copyright (c) 2002 , Murat Balaban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the EnderUNIX Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Fri Nov 22 07:44:00 EET 2002 aget-0.4/ChangeLog0100644000175000000000000000330607567343561013073 0ustar muratwheelEnderUNIX Aget ChangeLog * Fri Nov 22 07:44:20 EET 2002 Version 0.4 adds/changes the following: - Fixed bugs in Resume code. - Fixed miscalculations in the progress indicator * Fri Sep 20 18:11:11 EEST 2002 Version 0.39 adds/changes the following: - Hafiye is not 100 % POSIX compliant now. pread/pwrite functions are now used to simultaneously write to the local file. - Because of pwrite, the local file is not assembled at the end now, it's assembled at the same time. This speeds up large downloads; since previously lots of I/O was used to assemble the file. This is now decreased by at least fifty percent. - Solaris bug fixed. - Resume support. - Some more options: -p [port number to connect to] -l [local file name, if different from the remote file] -f [force use of suggested number of threads] - A progress indicator has been added. - Code has been re-arranged. Since aget starts to become larger, that was a need. - A README-Developer has been added for developers who want to contribute to aget. * Fri May 3 23:25:08 EEST 2002 Version 0.2p1 adds/changes the following: - getopt() problem is now solved, The program should really run on Posix systems. I've tested it on Linux, FreeBSD ans Solaris. - If no thread number is supplied, it is defaulted to 1. - A Solaris Makefile is added to the tarball. * Wed May 1 18:15:32 EEST 2002 Version 0.2 adds/changes the following: - File is now created seperately in each thread and later, it is assembled. Thus, some overhead from locking and unlocking of the mutex has been decreased. * Wed May 1 10:43:44 EEST 2002 First Public Release: 0.1 aget-0.4/Data.h0100644000175000000000000000147607566442007012343 0ustar muratwheel#ifndef DATA_H #define DATA_H #include #include #include "Defs.h" typedef struct request { char host[MAXHOSTSIZ]; /* Remote host */ char url[MAXURLSIZ]; /* URL */ char file[MAXBUFSIZ]; /* file name */ char lfile[MAXBUFSIZ]; /* if local file name is specified */ char ip[MAXIPSIZ]; /* Remote IP */ char username[MAXBUFSIZ]; char password[MAXBUFSIZ]; int port; int clength; /* Content-length */ unsigned char proto; /* Protocol */ } request; typedef struct thread_data { struct sockaddr_in sin; char getstr[GETREQSIZ]; long soffset; /* Start offset */ long foffset; /* Finish offset */ long offset; /* Current Offset */ long clength; /* Content Length */ int fd; pthread_t tid; /* Thread ID */ unsigned char status; /* thread exit status */ } thread_data; #endif aget-0.4/Defs.h0100644000175000000000000000125307567343561012352 0ustar muratwheel#ifndef DEFS_H #define DEFS_H enum { GETREQSIZ = 256, GETRECVSIZ = 8192, HEADREQSIZ = 512, MAXURLSIZ = 1024, MAXHOSTSIZ = 1024, MAXIPSIZ = 16, MAXBUFSIZ = 512, MAXTHREADS = 25, HTTPPORT = 80, UNKNOWNREQ = 2, FTPREQ = 21, PROTO_HTTP = 0xFF, PROTO_FTP = 0x00, STAT_OK = 0xFF, /* Download completed successfully */ STAT_INT = 0x0F, /* ^C caught, download interrupted */ STAT_ERR = 0x00 /* Download finished with error */ }; #define PROGVERSION "EnderUNIX Aget v0.4" #define HEADREQ "HEAD %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n\r\n" #define GETREQ "GET %s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nRange: bytes=%ld-\r\nConnection: close\r\n\r\n" #endif aget-0.4/Download.c0100644000175000000000000000553607566442007013235 0ustar muratwheel #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" #include "Download.h" extern sigset_t signal_set; extern int errno; unsigned int bwritten = 0; pthread_mutex_t bwritten_mutex = PTHREAD_MUTEX_INITIALIZER; void * http_get(void *arg) { struct thread_data *td; int sd; char *rbuf, *s; int dr, dw, i; long foffset; pthread_t tid; tid = pthread_self(); /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &signal_set, NULL); /* Set Cancellation Type to Asynchronous */ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); td = (struct thread_data *)arg; foffset = td->foffset; rbuf = (char *)calloc(GETRECVSIZ, sizeof(char)); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log(" socket creation failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((connect(sd, (const struct sockaddr *)&td->sin, sizeof(struct sockaddr))) == -1) { Log(" connection failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((send(sd, td->getstr, strlen(td->getstr), 0)) == -1) { Log(" send failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } if ((dr = recv(sd, rbuf, GETRECVSIZ, 0)) == -1) { Log(" recv failed: %s", tid, strerror(errno)); pthread_exit((void *)1); } handleHttpRetcode(rbuf); if ((strstr(rbuf, "HTTP/1.1 206")) == NULL) { fprintf(stderr, "Something unhandled happened, shutting down...\n"); exit(1); } s = rbuf; i = 0; while(1) { if (*s == '\n' && *(s - 1) == '\r' && *(s - 2) == '\n' && *(s - 3) == '\r') { s++; i++; break; } s++; i++; } td->offset = td->soffset; if ((dr - i ) > foffset) dw = pwrite(td->fd, s, (foffset - i), td->soffset); else dw = pwrite(td->fd, s, (dr - i), td->soffset); td->offset = td->soffset + dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); while (td->offset < foffset) { memset(rbuf, GETRECVSIZ, 0); dr = recv(sd, rbuf, GETRECVSIZ, 0); if ((td->offset + dr) > foffset) dw = pwrite(td->fd, rbuf, foffset - td->offset, td->offset); else dw = pwrite(td->fd, rbuf, dr, td->offset); td->offset += dw; pthread_mutex_lock(&bwritten_mutex); bwritten += dw; pthread_mutex_unlock(&bwritten_mutex); updateProgressBar(bwritten, td->clength); } if (td->offset == td->foffset) td->status = STAT_OK; /* Tell thet download is OK. */ close(sd); /* printf(" Part %d completed, leaving thread...\n", tid, td->tind);*/ pthread_exit(NULL); return NULL; } aget-0.4/Download.h0100644000175000000000000000013607566442010013223 0ustar muratwheel#ifndef DOWNLOAD_H #define DOWNLOAD_H #include void * http_get(void *); #endif aget-0.4/Head.c0100644000175000000000000000430107567357333012324 0ustar muratwheel #define _XOPEN_SOURCE 500 #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SOLARIS #include #include #endif #include "Head.h" #include "Data.h" #include "Defs.h" #include "Misc.h" extern int errno; extern int h_errno; void http_head_req(struct request *req) { struct sockaddr_in sin; struct hostent *he; int sd; char *sbuf; char *rbuf; char *tok; char *s; int clength; sbuf = (char *)calloc(HEADREQSIZ + strlen(req->url), sizeof(char)); rbuf = (char *)calloc(HEADREQSIZ, sizeof(char)); if ((he = gethostbyname(req->host)) == NULL) { Log("Error: Cannot resolve hostname for %s: %s", req->host, hstrerror(h_errno)); exit(1); } strncpy(req->ip, inet_ntoa(*(struct in_addr *)he->h_addr), MAXIPSIZ); time(&t_start); bzero(&sin, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr =inet_addr(req->ip); sin.sin_port = htons(req->port); if ((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { Log("Socket creation failed for Head Request: %s", strerror(errno)); exit(1); } if ((connect(sd, (const struct sockaddr *)&sin, sizeof(sin))) == -1) { Log("Connection failed for Head Request: %s", strerror(errno)); exit(1); } Log("Head-Request Connection established"); sprintf(sbuf, HEADREQ, req->url, req->host, PROGVERSION); if ((send(sd, sbuf, strlen(sbuf), 0)) == -1) { Log("send failed for Head Request: %s", strerror(errno)); exit(1); } if ((recv(sd, rbuf, HEADREQSIZ, 0)) == -1) { Log("recv failed for Head Request: %s", strerror(errno)); exit(1); } handleHttpRetcode(rbuf); tok = strtok(rbuf, "\r\n"); if ((strstr(tok, "HTTP/1.1 200")) != NULL) { while ((tok = strtok(NULL, "\r\n")) != NULL) { if ((strstr(tok, "Content-Length")) != NULL) { s = (tok + strlen("Content-Length: ")); clength = atoi(s); req->clength = clength; } } } free(sbuf); free(rbuf); } aget-0.4/Head.h0100644000175000000000000000014007566442007012316 0ustar muratwheel#ifndef HEAD_H #define HEAD_H #include "Data.h" void http_head_req(struct request *); #endif aget-0.4/INSTALL0100644000175000000000000000123307567343561012347 0ustar muratwheelEnderUNIX Aget v0.4 INSTALL -------------------------- The program is tested on RedHat Linux 7.2 (gcc-2.96), FreeBSD 4.5 and Solaris 8. For Solaris, you'll have to use Makefile.Solaris. To compile Aget, type: # make && make strip To install the compiled binary, type: # make install and binary will be installed as /usr/local/bin/aget You're done. All you need is a file to download and supply that url and desired number of segments to aget, e.g: $ aget -n10 http://www.enderunix.org/murat/barismanco-hibrahimsofrasi.mp3 Aget will split the file into 10 segments and will download it for you. Fri Nov 22 07:49:15 EET 2002 http://www.enderunix.org/aget/ aget-0.4/Makefile0100644000175000000000000000057407567357454012773 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -Wall -W -pedantic LDFLAGS = -pthread CC = gcc STRIP = strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ c: rm -f *core* *~ *log aget-0.4/Makefile.FreeBSD0100644000175000000000000000057407567357333014200 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -g -Wall -W -pedantic LDFLAGS = -pthread CC = gcc STRIP = strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ c: rm -f *core* *~ *log aget-0.4/Makefile.Solaris0100644000175000000000000000061607567357333014377 0ustar muratwheel# EnderUNIX Aget Makefile # http://www.enderunix.org/aget/ OBJS = main.o Aget.o Misc.o Head.o Signal.o Download.o Resume.o CFLAGS = -Wall -W -pedantic -DSOLARIS LDFLAGS = -lpthread -lsocket -lnsl -lresolv CC = gcc STRIP=/usr/local/bin/strip all: $(OBJS) $(CC) -o aget $(OBJS) $(LDFLAGS) strip: $(all) $(STRIP) aget install: cp -f aget /usr/local/bin/aget clean: rm -f aget *.o core.* *~ aget-0.4/Misc.c0100644000175000000000000001105407566442007012351 0ustar muratwheel#include #include #include #include #include #include #include #include #include #include "Misc.h" #include "Data.h" void parse_url(char *url, struct request *req) { char *s; int i, j, k; i = j = k = 0; s = url; if ((strncmp(url, "ftp://", 6)) == 0) { fprintf(stderr, "Error: Currently Aget doesn't support FTP requests...\n"); exit(1); } else if ((strncmp(url, "http://", 7)) != 0) { fprintf(stderr, "Error: URL should be of the form http://...\n"); exit(1); } if (req->port == 0) { req->port = 80; req->proto = PROTO_HTTP; } s = url + 7; /* Jump pass http:// part */ for (i = 0; *s != '/'; i++, s++) { if (i > MAXHOSTSIZ) { fprintf(stderr, "Error: Cannot get hostname from URL...\n"); exit(1); } if (*s == ':') { /* If user/pass is supplied like; http://murat:12345@x.y.com/url.html */ while(*s != '/') { req->username[j++] = *--s; i--; } req->username[--j] = '\0'; revstr(req->username); while(1) { if (*s == ':') { while(*s != '@') { if (k > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get password from URL...\n"); exit(1); } req->password[k++] = *++s; } break; } s++; } req->password[--k] = '\0'; } req->host[i] = *s; } req->host[i] = '\0'; for (i = 0; *s != '\0'; i++, s++) { if (i > MAXURLSIZ) { fprintf(stderr, "Error: Cannot get remote file name from URL...\n"); exit(1); } req->url[i] = *s; } req->url[i] = '\0'; --s; for (i = 0; *s != '/'; i++, s--) { if (i > MAXBUFSIZ) { fprintf(stderr, "Error: Cannot get local file name from URL...\n"); exit(1); } req->file[i] = *s; } req->file[i] = '\0'; revstr(req->file); } int numofthreads(int size) { if (size == 0) return 0; else if (size < BUFSIZ * 2) /* < 16384 */ return 1; else if ((size >= BUFSIZ * 2) && (size < BUFSIZ * 4)) /* 16384 < x < 32678 */ return 2; else if ((size >= BUFSIZ * 4) && (size < BUFSIZ * 8)) /* 32768 < x < 65536 */ return 3; else if ((size >= BUFSIZ * 8) && (size < BUFSIZ * 16)) /* 65536 < x < 131072 */ return 4; else if ((size >= BUFSIZ * 16) && (size < BUFSIZ * 32)) /* 131072 < x < 262144 */ return 5; else if ((size >= BUFSIZ * 32) && (size < BUFSIZ * 64)) return 6; else if ((size >= BUFSIZ * 64) && (size < BUFSIZ * 128)) return 7; else if ((size >= BUFSIZ * 128) && (size < BUFSIZ * 256)) return 8; else if ((size >= BUFSIZ * 256) && (size < BUFSIZ * 512)) return 9; else return 10; } int calc_offset(int total, int part, int nthreads) { return (part * (total / nthreads)); } void usage() { fprintf(stderr, "usage: aget [options] url\n"); fprintf(stderr, "\toptions:\n"); fprintf(stderr, "\t\t-p port number\n"); fprintf(stderr, "\t\t-l local file name\n"); fprintf(stderr, "\t\t-n suggested number of threads\n"); fprintf(stderr, "\t\t-f force using suggested number of threads\n"); fprintf(stderr, "\t\t-h this screen\n"); fprintf(stderr, "\t\t-v version info\n"); fprintf(stderr, "\n"); fprintf(stderr, "http//www.enderunix.org/aget/\n"); } /* reverse a given string */ void revstr(char *str) { char *p, *s; int i; int size; if ((size = strlen(str)) == 0) return; p = (char *)calloc(size, sizeof(char)); s = p; for (i = size; i >= 0; i--, s++) *s = *(str + i - 1); *s = '\0'; memset(str, 0, size); strncpy(str, p, size); free(p); } /* Log */ void Log(char *fmt, ...) { va_list ap; char *lfmt; lfmt = (char *)calloc(7 + strlen(fmt), sizeof(char)); sprintf(lfmt, " %s", fmt); fflush(stdout); va_start(ap, fmt); vfprintf(stderr, lfmt, ap); va_end(ap); if (fmt[0] != '\0' && fmt[strlen(fmt) - 1] == ':') fprintf(stderr, " %s", strerror(errno)); fprintf(stderr, "\n"); free(lfmt); } /* Progress Bar */ void updateProgressBar(float cur, float tot) { float rat; int ndot, i; static float prev = -1; rat = cur/tot; ndot = (int)(rat * 100); if ((ndot < prev + 5) && (ndot != 100)) return; for (i = 0; i < ndot; i += 2) putchar('.'); for (i = ndot - 1; i < 100; i += 2) putchar(' '); printf("[%d%% completed]\n", ndot); prev = ndot; } void handleHttpRetcode(char *rbuf) { if ((strstr(rbuf, "HTTP/1.1 416")) != NULL) { Log("Server returned HTTP/1.1 416 - Requested Range Not Satisfiable\n"); exit(1); } else if ((strstr(rbuf, "HTTP/1.1 403")) != NULL) { Log(" * * See COPYING for copyright and copying restrictions * */ #ifndef MISC_H #define MISC_H #include "Data.h" #define LOGSIZ 1024 int calc_offset(int, int, int); int numofthreads(int); void parse_url(char *, struct request *); void usage(); void revstr(char *); /* Reverse String */ void Log(char *, ...); /* Log */ void updateProgressBar(float, float); void handleHttpRetcode(char *); time_t t_start, t_finish; #endif aget-0.4/README0100644000175000000000000000074507567343561012205 0ustar muratwheelEnderUNIX Aget v0.4 README -------------------------- This program is a starting point for a very useful project like FlashGet for Win32. My aim is to provide all the functionality that program has. If you need assistance in installing Aget, you can see INSTALL file, Program is tested on RedHat Linux 8, FreeBSD 4.7 and Solaris 7,8 (Sparc). It proved to be successful. You can get much more info from the Aget Web site: http://www.enderunix.org/aget Fri Nov 22 07:50:38 EET 2002 aget-0.4/README-Developer0100644000175000000000000000274207567343561014127 0ustar muratwheelOk, Here's a developer's primer for aget source: The code is splitted into several files: main.{c|h} * starts a thread for handling signals. * gets program options and decides the program's behviour by setting a few global variables. Aget.{c|h} * get() and resume_get() functions starts the download process. Depending on the protocol, necessary methods are called. resume_get() is for resume jobs. If the there is a log file of a previous job, the values are taken from it. Head.{c|h} * Before the actual download, a head request is sent to the server to determine the current status of the file. Some values like: 1. HTTP return code (e.g. 404, 206, 403, 200) 2. Content-length might be of interest there. Download.{c|h} * Aget is no more %100 Posix compliant :( pwrite is from X_OPEN standarts. I had to do it, since I needed a way to simultaneously write to the same file without loosing any time while synchronising the threads. For the time, there is only http_get. When aget is ftp-aware ftp_get will be in this file. Resume.{c|h} * Methods about saving and restoring history files are in this file. Signal.{c|h} * Signal Handling functions. Misc.{c|h} * Other functions which do not fit any of the general categories are stored here. These are generally helper functions... For more information read the aget source, and you may contact murat at enderunix dot org. Fri Nov 22 07:51:04 EET 2002 aget-0.4/Resume.c0100644000175000000000000000367207566714167012736 0ustar muratwheel #include #include #include #include #include #include "Data.h" #include "Resume.h" #include "Misc.h" extern struct thread_data *wthread; extern struct request *req; extern int nthreads; extern int bwritten; void save_log() { char *logfile; struct hist_data h; FILE *fp; logfile = (char *)calloc(255, sizeof(char)); if (strlen(req[0].lfile) == 0) snprintf(logfile, 255, "aget-%s.log", req[0].file); else snprintf(logfile, 255, "aget-%s.log", req[0].lfile); if ((fp = fopen(logfile, "w")) == NULL) { fprintf(stderr, "cannot open log file %s for writing: %s\n", logfile, strerror(errno)); exit(1); } memcpy(&(h.req), req, sizeof(struct request)); memcpy(&(h.wthread), wthread, sizeof(struct thread_data) * nthreads); h.nthreads = nthreads; h.bwritten = bwritten; printf("--> Logfile is: %s, so far %d bytes have been transferred\n", logfile, h.bwritten); fwrite(&h, sizeof(struct hist_data), 1, fp); fclose(fp); free(logfile); } int read_log(struct hist_data *h) { char *logfile; FILE *fp; logfile = (char *)calloc(255, sizeof(char)); if (strlen(req[0].lfile) == 0) snprintf(logfile, 255, "aget-%s.log", req[0].file); else snprintf(logfile, 255, "aget-%s.log", req[0].lfile); Log("Attempting to read log file %s for resuming download job...", logfile); if ((fp = fopen(logfile, "r")) == NULL) { if (errno == ENOENT) { Log("Couldn't find log file for this download, starting a clean job..."); return -1; } else { fprintf(stderr, "cannot open log file %s for reading: %s\n", logfile, strerror(errno)); exit(1); } } fread(h, sizeof(struct hist_data), 1, fp); bwritten = h->bwritten; fclose(fp); Log("%d bytes already transferred", bwritten); /* Unlinking logfile after we've read it */ if ((unlink(logfile)) == -1) fprintf(stderr, "read_log: cannot remove stale log file %s: %s\n", logfile, strerror(errno)); free(logfile); return 0; } aget-0.4/Resume.h0100644000175000000000000000041107566714167012727 0ustar muratwheel#ifndef RESUME_H #define RESUME_H #include "Data.h" #include "Defs.h" typedef struct hist_data { struct request req; int nthreads; int bwritten; struct thread_data wthread[MAXTHREADS]; } hist_data; void save_log(); int read_log(struct hist_data *); #endif aget-0.4/Signal.c0100644000175000000000000000201407567357333012677 0ustar muratwheel #include #include #include #include #include #include "Signal.h" #include "Data.h" #include "Resume.h" #include "Misc.h" extern int nthreads; extern struct thread_data *wthread; extern struct request *req; extern int bwritten; extern pthread_mutex_t bwritten_mutex; void * signal_waiter(void *arg) { int signal; arg = NULL; pthread_sigmask(SIG_UNBLOCK, &signal_set, NULL); while(1) { #ifdef SOLARIS sigwait(&signal_set); #else sigwait(&signal_set, &signal); #endif switch(signal) { case SIGINT: sigint_handler(); break; case SIGALRM: sigalrm_handler(); break; } } } void sigint_handler(void) { int i; printf("^C caught, saving download job...\n"); for (i = 0; i < nthreads; i++) { pthread_cancel(wthread[i].tid); wthread[i].status &= STAT_INT; /* Interrupted download */ } save_log(); exit(0); } void sigalrm_handler(void) { printf("Signal Alarm came\n"); updateProgressBar(bwritten, req->clength); alarm(1); } aget-0.4/Signal.h0100644000175000000000000000030307566442007012673 0ustar muratwheel#ifndef SIGNAL_H #define SIGNAL_H #include #include sigset_t signal_set; void * signal_waiter(void *arg); void sigint_handler(void); void sigalrm_handler(void); #endif aget-0.4/THANKS0100644000175000000000000000033307567343561012231 0ustar muratwheel Sensei, and my EnderUNIX team members... Atilim Boy nikinakof Oleg Mitsura Fri Nov 22 07:51:36 EET 2002 aget-0.4/TODO0100644000175000000000000000062107567343561012006 0ustar muratwheelEnderUNIX Aget v0.4 TODO: ------------------------- -> HTTP Proxy support -> Constructing libaget.so/libaget.a for IstanbulX project of acikkod.org -> Thread exit status evaluation: If one of the threads fail, the program should retry the failed later on. -> FreeBSD/NetBSD/OpenBSD Port/Package -> Multiple downloads -> FTP Support Fri Nov 22 07:55:26 EET 2002 http://www.enderunix.org/aget/ aget-0.4/main.c0100644000175000000000000000465107667330341012406 0ustar muratwheel#include #include #include #include #include #include #include "Defs.h" #include "Data.h" #include "Misc.h" #include "Aget.h" #include "Signal.h" #include "Resume.h" #include "main.h" extern int errno; int main(int argc, char **argv) { extern char *optarg; extern int optind; int c, error = 0, ret; struct hist_data h; int retlog; /* Allocate heap for download request * struct request stores all the information that might be * of interest */ req = (struct request *)calloc(1, sizeof(struct request)); /* Only some signals will be emitted */ sigemptyset(&signal_set); sigaddset(&signal_set, SIGINT); sigaddset(&signal_set, SIGALRM); /* Block out all signals */ pthread_sigmask(SIG_BLOCK, &signal_set, NULL); /* Create a thread for hadling signals */ if ((ret = pthread_create(&hthread, NULL, signal_waiter, NULL)) != 0) { fprintf(stderr, "main: cannot create signal_waiter thread: %s, exiting...\n", strerror(errno)); exit(-1); } while (!error && (c = getopt(argc,argv,"p:l:n:hfv")) != -1) { switch(c) { case 'p': req->port = atoi(optarg); break; case 'f': fsuggested = 1; break; case 'l': strncpy(req->lfile, optarg, MAXBUFSIZ); break; case 'n': if ((nthreads = atoi(optarg)) > MAXTHREADS) { Log("Error: Maximum # of threads allowed is %d\n", MAXTHREADS); nthreads = 0; } break; case 'h': printf("%s\n", PROGVERSION); usage(); exit(0); break; case 'v': printf("%s\nby Murat BALABAN \n", PROGVERSION); exit(0); break; default: error = 1; usage(); exit(1); break; } } if (error) { usage(); exit(1); } if (fsuggested == 1 && nthreads == 0) { fprintf(stderr, "\nERROR: -f and -n should be used together!, exiting...\n\n"); usage(); exit(1); } if (argc == 2) /* If only url is supplied... */ fullurl = strdup(argv[1]); else if (optind < argc) if (argc > 2) fullurl = strdup(argv[optind]); else { usage(); exit(1); } else if (optind == argc) { usage(); exit(1); } parse_url(fullurl, req); /* If a log file for a previous try has been found, read it and * resume the download job (resume_get), otherwise, start with * a clean job (get) * * Logfile is of the pattern: aget-$file_name.log */ if ((retlog = read_log(&h)) != -1) resume_get(&h); else get(req); return 0; } aget-0.4/main.h0100644000175000000000000000042507566442007012407 0ustar muratwheel#ifndef MAIN_H #define MAIN_H #include #include "Data.h" char *fullurl; int nthreads; int fsuggested = 0; struct request *req; /* Download jobs */ pthread_t hthread; /* Helper thread for signals */ struct thread_data *wthread; /* Worker Threads */ #endif