diff --git a/service/extra/quickdown b/service/extra/quickdown deleted file mode 160000 index 8070d96d..00000000 --- a/service/extra/quickdown +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8070d96ddcc7680c54848299cc10379005d7ffdf diff --git a/service/extra/quickdown/LICENSE b/service/extra/quickdown/LICENSE new file mode 100644 index 00000000..02d29c70 --- /dev/null +++ b/service/extra/quickdown/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 xlzd + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/service/extra/quickdown/README.md b/service/extra/quickdown/README.md new file mode 100644 index 00000000..75c00a03 --- /dev/null +++ b/service/extra/quickdown/README.md @@ -0,0 +1,2 @@ +# quickdown +A Golang package for quickly download. diff --git a/service/extra/quickdown/downloadtask.go b/service/extra/quickdown/downloadtask.go new file mode 100644 index 00000000..c4e92da1 --- /dev/null +++ b/service/extra/quickdown/downloadtask.go @@ -0,0 +1,198 @@ +package quickdown + +import ( + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "strconv" + "sync" + "time" + + "gopkg.in/cheggaaa/pb.v1" +) + +type DownloadTask struct { + url string + filename string + + workerCount int + workers []*worker + + headOnce sync.Once + contentLength int + wg sync.WaitGroup + done chan bool +} + +func NewDefaultDownloadTask(url string) *DownloadTask { + return NewDownloadTask(url, "", 5) +} + +func NewDownlosadTaskWithWorkers(url string, workerCount int) *DownloadTask { + return NewDownloadTask(url, "", workerCount) +} + +func NewDownloadTask(url, filename string, workerCount int) *DownloadTask { + if workerCount < 1 { + panic(errors.New("[DownloadTask] worker count must greater than 0.")) + } + + if filename == "" { + filename = path.Base(url) + } + + return &DownloadTask{ + url: url, + workerCount: workerCount, + filename: filename, + done: make(chan bool), + } +} + +func (self *DownloadTask) getContentLength() int { + self.headOnce.Do(func() { + request, _ := http.NewRequest("HEAD", self.url, nil) + request.Header.Add("User-Agent", "github.com/xlzd/godown") + response, err := client.Do(request) + if err != nil { + msg := fmt.Sprintf("[DownloadTask] Check %s head fail.", self.url) + panic(errors.New(msg)) + } + + if contentLengthStr := response.Header.Get("Content-Length"); contentLengthStr != "" { + self.contentLength, _ = strconv.Atoi(contentLengthStr) + } else { + log.Print(fmt.Sprintf("[DownloadTask] Get %s content length fail, download via one gorouting.", self.url)) + } + }) + return self.contentLength +} + +func (self *DownloadTask) initWorkers() { + cl := self.getContentLength() + if cl <= 4096 { + self.workerCount = 1 + log.Println("[DownloadTask] content length too small, use one goroutine.") + } else if cl < self.workerCount*4096 { + self.workerCount = (cl + 4095) / 4096 + log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount) + } else { + log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount) + } + + persize := (cl + self.workerCount - 1) / self.workerCount + self.wg.Add(self.workerCount) + for i := 0; i < self.workerCount; i++ { + min := i * persize + max := (i+1)*persize - 1 + if max > cl { + max = cl + } + filename := fmt.Sprintf("%s.part.%d.%d", self.filename, self.workerCount, i) + self.workers = append(self.workers, newWorker(i, self.url, min, max, &self.wg, filename)) + } +} + +func (self *DownloadTask) merge() { + err := os.Rename(self.workers[0].filename, self.filename) + if err != nil { + panic(err) + } + + if self.workerCount == 1 { + return + } + + out, err := os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + panic(err) + } + defer out.Close() + + for _, w := range self.workers[1:] { + in, err := os.Open(w.filename) + if err != nil { + panic(err) + } + if _, err = io.Copy(out, in); err != nil { + panic(err) + } + in.Close() + out.Sync() + os.Remove(w.filename) + } +} + +func (self *DownloadTask) wait() { + self.wg.Wait() + close(self.done) +} + +func (self *DownloadTask) showProgress() { + pbs := make([]*pb.ProgressBar, self.workerCount) + full := 10000 + + for i := range self.workers { + bar := pb.New(full).Prefix(fmt.Sprintf("\033[0;32m[Worker %d]\033[0m", i+1)) + //bar.Format("[=>~]") + bar.ShowCounters = false + bar.ShowFinalTime = true + bar.ShowElapsedTime = true + //bar.AlwaysUpdate = true + //bar.SetRefreshRate(time.Millisecond * 100) + pbs[i] = bar + } + + pool, err := pb.StartPool(pbs...) + if err != nil { + panic(err) + } + + for { + select { + case <-self.done: + for _, p := range pbs { + if !p.IsFinished() { + p.Set(full) + p.Finish() + } + } + pool.Stop() + return + case <-time.After(time.Millisecond * 200): + for i, w := range self.workers { + if w.done { + if !pbs[i].IsFinished() { + pbs[i].Set(full) + pbs[i].Finish() + } + } else { + current := int(float64(full) * float64(w.downloaded) / float64(w.total)) + pbs[i].Set(current) + } + } + } + } +} + +func (self *DownloadTask) Download() { + select { + case <-self.done: + panic(errors.New("[DownloadTask] already downloaded.")) + default: + } + + self.initWorkers() + for _, w := range self.workers { + go w.run() + } + + go self.showProgress() + self.wait() + self.merge() + log.Printf("[DownloadTask] download %s done.", self.url) +} diff --git a/service/extra/quickdown/func.go b/service/extra/quickdown/func.go new file mode 100644 index 00000000..248c2fd3 --- /dev/null +++ b/service/extra/quickdown/func.go @@ -0,0 +1,51 @@ +package quickdown + +import ( + "errors" + "log" + "net/http" +) + +var client *http.Client + +func Download(url string) error { + return download(url, "", 5) +} + +func DownloadTo(url, filename string) error { + return download(url, filename, 5) +} + +func DownloadWithWorkers(url string, workerCount int) error { + return download(url, "", workerCount) +} + +func DownloadWithWorkersTo(url string, workerCount int, filename string) error { + return download(url, filename, workerCount) +} + +func download(url, filename string, workerCount int) (err error) { + defer func() { + if r := recover(); r != nil { + switch x := r.(type) { + case string: + err = errors.New(x) + case error: + err = x + default: + err = errors.New("Unknown panic") + } + log.Println(err) + } + }() + NewDownloadTask(url, filename, workerCount).Download() + return +} + +func SetHttpClient(c *http.Client) { + client = c +} + +func GetHttpClient() (c *http.Client) { + return client +} diff --git a/service/extra/quickdown/worker.go b/service/extra/quickdown/worker.go new file mode 100644 index 00000000..f417afca --- /dev/null +++ b/service/extra/quickdown/worker.go @@ -0,0 +1,92 @@ +package quickdown + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "sync" +) + +type worker struct { + id int + url string + filename string + min, max int + downloaded, total int + done bool + wg *sync.WaitGroup +} + +func newWorker(id int, url string, min, max int, wg *sync.WaitGroup, filename string) *worker { + return &worker{ + id: id, + url: url, + min: min, + max: max, + total: max - min, + wg: wg, + filename: filename, + } +} + +func (self *worker) run() { + if self.done { + panic(errors.New(fmt.Sprintf("worker(id=%d, url=%s) already done.", self.id, self.url))) + } + + var tmpfile *os.File + fInfo, err := os.Stat(self.filename) + if err == nil { + fSize := int(fInfo.Size()) + self.min += fSize + self.downloaded = fSize + + if fSize >= self.total { + self.setDone() + return + } + + tmpfile, err = os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600) + } else { + tmpfile, _ = os.Create(self.filename) + } + defer tmpfile.Close() + + request, _ := http.NewRequest("GET", self.url, nil) + request.Header.Add("User-Agent", "github.com/xlzd/godown") + if self.min >= 0 && self.max > 0 { + request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", self.min, self.max)) + } + + response, err := client.Do(request) + if err != nil { + panic(err) + } + defer response.Body.Close() + + for i := 0; ; i++ { + buf := make([]byte, 4096*16) + n, err := response.Body.Read(buf) + if n != 0 { + self.downloaded += n + tmpfile.Write(buf[:n]) + } + + if err != nil { + if err == io.EOF { + break + } + panic(err) + } + } + + tmpfile.Sync() + self.setDone() +} + +func (self *worker) setDone() { + self.done = true + self.wg.Done() +}