chore: git rm -rf . --cached

This commit is contained in:
mzz2017 committed 2019-11-03 05:25:52 +08:00
1 parent eaf2fff9ec
commit ff205bbd2e
6 files changed
+364 -1

No files matched your search

Submodule service/extra/quickdown deleted from 8070d96ddc.
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018 xlzd
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+2
View File
@@ -0,0 +1,2 @@
# quickdown
A Golang package for quickly download.
+198
View File
@@ -0,0 +1,198 @@
package quickdown
import (
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"path"
"strconv"
"sync"
"time"
"gopkg.in/cheggaaa/pb.v1"
)
type DownloadTask struct {
url string
filename string
workerCount int
workers []*worker
headOnce sync.Once
contentLength int
wg sync.WaitGroup
done chan bool
}
func NewDefaultDownloadTask(url string) *DownloadTask {
return NewDownloadTask(url, "", 5)
}
func NewDownlosadTaskWithWorkers(url string, workerCount int) *DownloadTask {
return NewDownloadTask(url, "", workerCount)
}
func NewDownloadTask(url, filename string, workerCount int) *DownloadTask {
if workerCount < 1 {
panic(errors.New("[DownloadTask] worker count must greater than 0."))
}
if filename == "" {
filename = path.Base(url)
}
return &DownloadTask{
url: url,
workerCount: workerCount,
filename: filename,
done: make(chan bool),
}
}
func (self *DownloadTask) getContentLength() int {
self.headOnce.Do(func() {
request, _ := http.NewRequest("HEAD", self.url, nil)
request.Header.Add("User-Agent", "github.com/xlzd/godown")
response, err := client.Do(request)
if err != nil {
msg := fmt.Sprintf("[DownloadTask] Check %s head fail.", self.url)
panic(errors.New(msg))
}
if contentLengthStr := response.Header.Get("Content-Length"); contentLengthStr != "" {
self.contentLength, _ = strconv.Atoi(contentLengthStr)
} else {
log.Print(fmt.Sprintf("[DownloadTask] Get %s content length fail, download via one gorouting.", self.url))
}
})
return self.contentLength
}
func (self *DownloadTask) initWorkers() {
cl := self.getContentLength()
if cl <= 4096 {
self.workerCount = 1
log.Println("[DownloadTask] content length too small, use one goroutine.")
} else if cl < self.workerCount*4096 {
self.workerCount = (cl + 4095) / 4096
log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount)
} else {
log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount)
}
persize := (cl + self.workerCount - 1) / self.workerCount
self.wg.Add(self.workerCount)
for i := 0; i < self.workerCount; i++ {
min := i * persize
max := (i+1)*persize - 1
if max > cl {
max = cl
}
filename := fmt.Sprintf("%s.part.%d.%d", self.filename, self.workerCount, i)
self.workers = append(self.workers, newWorker(i, self.url, min, max, &self.wg, filename))
}
}
func (self *DownloadTask) merge() {
err := os.Rename(self.workers[0].filename, self.filename)
if err != nil {
panic(err)
}
if self.workerCount == 1 {
return
}
out, err := os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
panic(err)
}
defer out.Close()
for _, w := range self.workers[1:] {
in, err := os.Open(w.filename)
if err != nil {
panic(err)
}
if _, err = io.Copy(out, in); err != nil {
panic(err)
}
in.Close()
out.Sync()
os.Remove(w.filename)
}
}
func (self *DownloadTask) wait() {
self.wg.Wait()
close(self.done)
}
func (self *DownloadTask) showProgress() {
pbs := make([]*pb.ProgressBar, self.workerCount)
full := 10000
for i := range self.workers {
bar := pb.New(full).Prefix(fmt.Sprintf("\033[0;32m[Worker %d]\033[0m", i+1))
//bar.Format("[=>~]")
bar.ShowCounters = false
bar.ShowFinalTime = true
bar.ShowElapsedTime = true
//bar.AlwaysUpdate = true
//bar.SetRefreshRate(time.Millisecond * 100)
pbs[i] = bar
}
pool, err := pb.StartPool(pbs...)
if err != nil {
panic(err)
}
for {
select {
case <-self.done:
for _, p := range pbs {
if !p.IsFinished() {
p.Set(full)
p.Finish()
}
}
pool.Stop()
return
case <-time.After(time.Millisecond * 200):
for i, w := range self.workers {
if w.done {
if !pbs[i].IsFinished() {
pbs[i].Set(full)
pbs[i].Finish()
}
} else {
current := int(float64(full) * float64(w.downloaded) / float64(w.total))
pbs[i].Set(current)
}
}
}
}
}
func (self *DownloadTask) Download() {
select {
case <-self.done:
panic(errors.New("[DownloadTask] already downloaded."))
default:
}
self.initWorkers()
for _, w := range self.workers {
go w.run()
}
go self.showProgress()
self.wait()
self.merge()
log.Printf("[DownloadTask] download %s done.", self.url)
}
+51
View File
@@ -0,0 +1,51 @@
package quickdown
import (
"errors"
"log"
"net/http"
)
var client *http.Client
func Download(url string) error {
return download(url, "", 5)
}
func DownloadTo(url, filename string) error {
return download(url, filename, 5)
}
func DownloadWithWorkers(url string, workerCount int) error {
return download(url, "", workerCount)
}
func DownloadWithWorkersTo(url string, workerCount int, filename string) error {
return download(url, filename, workerCount)
}
func download(url, filename string, workerCount int) (err error) {
defer func() {
if r := recover(); r != nil {
switch x := r.(type) {
case string:
err = errors.New(x)
case error:
err = x
default:
err = errors.New("Unknown panic")
}
log.Println(err)
}
}()
NewDownloadTask(url, filename, workerCount).Download()
return
}
func SetHttpClient(c *http.Client) {
client = c
}
func GetHttpClient() (c *http.Client) {
return client
}
+92
View File
@@ -0,0 +1,92 @@
package quickdown
import (
"errors"
"fmt"
"io"
"net/http"
"os"
"sync"
)
type worker struct {
id int
url string
filename string
min, max int
downloaded, total int
done bool
wg *sync.WaitGroup
}
func newWorker(id int, url string, min, max int, wg *sync.WaitGroup, filename string) *worker {
return &worker{
id: id,
url: url,
min: min,
max: max,
total: max - min,
wg: wg,
filename: filename,
}
}
func (self *worker) run() {
if self.done {
panic(errors.New(fmt.Sprintf("worker(id=%d, url=%s) already done.", self.id, self.url)))
}
var tmpfile *os.File
fInfo, err := os.Stat(self.filename)
if err == nil {
fSize := int(fInfo.Size())
self.min += fSize
self.downloaded = fSize
if fSize >= self.total {
self.setDone()
return
}
tmpfile, err = os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600)
} else {
tmpfile, _ = os.Create(self.filename)
}
defer tmpfile.Close()
request, _ := http.NewRequest("GET", self.url, nil)
request.Header.Add("User-Agent", "github.com/xlzd/godown")
if self.min >= 0 && self.max > 0 {
request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", self.min, self.max))
}
response, err := client.Do(request)
if err != nil {
panic(err)
}
defer response.Body.Close()
for i := 0; ; i++ {
buf := make([]byte, 4096*16)
n, err := response.Body.Read(buf)
if n != 0 {
self.downloaded += n
tmpfile.Write(buf[:n])
}
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
}
tmpfile.Sync()
self.setDone()
}
func (self *worker) setDone() {
self.done = true
self.wg.Done()
}