mirror of
https://github.com/v2rayA/v2rayA.git
synced 2026-11-20 23:02:25 +03:00
chore: git rm -rf . --cached
This commit is contained in:
1 parent
eaf2fff9ec
commit
ff205bbd2e
6 files changed
+364
-1
No files matched your search
Submodule service/extra/quickdown deleted from 8070d96ddc.
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 xlzd
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,2 @@
|
||||
# quickdown
|
||||
A Golang package for quickly download.
|
||||
@@ -0,0 +1,198 @@
|
||||
package quickdown
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gopkg.in/cheggaaa/pb.v1"
|
||||
)
|
||||
|
||||
type DownloadTask struct {
|
||||
url string
|
||||
filename string
|
||||
|
||||
workerCount int
|
||||
workers []*worker
|
||||
|
||||
headOnce sync.Once
|
||||
contentLength int
|
||||
wg sync.WaitGroup
|
||||
done chan bool
|
||||
}
|
||||
|
||||
func NewDefaultDownloadTask(url string) *DownloadTask {
|
||||
return NewDownloadTask(url, "", 5)
|
||||
}
|
||||
|
||||
func NewDownlosadTaskWithWorkers(url string, workerCount int) *DownloadTask {
|
||||
return NewDownloadTask(url, "", workerCount)
|
||||
}
|
||||
|
||||
func NewDownloadTask(url, filename string, workerCount int) *DownloadTask {
|
||||
if workerCount < 1 {
|
||||
panic(errors.New("[DownloadTask] worker count must greater than 0."))
|
||||
}
|
||||
|
||||
if filename == "" {
|
||||
filename = path.Base(url)
|
||||
}
|
||||
|
||||
return &DownloadTask{
|
||||
url: url,
|
||||
workerCount: workerCount,
|
||||
filename: filename,
|
||||
done: make(chan bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (self *DownloadTask) getContentLength() int {
|
||||
self.headOnce.Do(func() {
|
||||
request, _ := http.NewRequest("HEAD", self.url, nil)
|
||||
request.Header.Add("User-Agent", "github.com/xlzd/godown")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("[DownloadTask] Check %s head fail.", self.url)
|
||||
panic(errors.New(msg))
|
||||
}
|
||||
|
||||
if contentLengthStr := response.Header.Get("Content-Length"); contentLengthStr != "" {
|
||||
self.contentLength, _ = strconv.Atoi(contentLengthStr)
|
||||
} else {
|
||||
log.Print(fmt.Sprintf("[DownloadTask] Get %s content length fail, download via one gorouting.", self.url))
|
||||
}
|
||||
})
|
||||
return self.contentLength
|
||||
}
|
||||
|
||||
func (self *DownloadTask) initWorkers() {
|
||||
cl := self.getContentLength()
|
||||
if cl <= 4096 {
|
||||
self.workerCount = 1
|
||||
log.Println("[DownloadTask] content length too small, use one goroutine.")
|
||||
} else if cl < self.workerCount*4096 {
|
||||
self.workerCount = (cl + 4095) / 4096
|
||||
log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount)
|
||||
} else {
|
||||
log.Printf("[DownloadTask] content length %d, use %d goroutine.\n", cl, self.workerCount)
|
||||
}
|
||||
|
||||
persize := (cl + self.workerCount - 1) / self.workerCount
|
||||
self.wg.Add(self.workerCount)
|
||||
for i := 0; i < self.workerCount; i++ {
|
||||
min := i * persize
|
||||
max := (i+1)*persize - 1
|
||||
if max > cl {
|
||||
max = cl
|
||||
}
|
||||
filename := fmt.Sprintf("%s.part.%d.%d", self.filename, self.workerCount, i)
|
||||
self.workers = append(self.workers, newWorker(i, self.url, min, max, &self.wg, filename))
|
||||
}
|
||||
}
|
||||
|
||||
func (self *DownloadTask) merge() {
|
||||
err := os.Rename(self.workers[0].filename, self.filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if self.workerCount == 1 {
|
||||
return
|
||||
}
|
||||
|
||||
out, err := os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
for _, w := range self.workers[1:] {
|
||||
in, err := os.Open(w.filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err = io.Copy(out, in); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
in.Close()
|
||||
out.Sync()
|
||||
os.Remove(w.filename)
|
||||
}
|
||||
}
|
||||
|
||||
func (self *DownloadTask) wait() {
|
||||
self.wg.Wait()
|
||||
close(self.done)
|
||||
}
|
||||
|
||||
func (self *DownloadTask) showProgress() {
|
||||
pbs := make([]*pb.ProgressBar, self.workerCount)
|
||||
full := 10000
|
||||
|
||||
for i := range self.workers {
|
||||
bar := pb.New(full).Prefix(fmt.Sprintf("\033[0;32m[Worker %d]\033[0m", i+1))
|
||||
//bar.Format("[=>~]")
|
||||
bar.ShowCounters = false
|
||||
bar.ShowFinalTime = true
|
||||
bar.ShowElapsedTime = true
|
||||
//bar.AlwaysUpdate = true
|
||||
//bar.SetRefreshRate(time.Millisecond * 100)
|
||||
pbs[i] = bar
|
||||
}
|
||||
|
||||
pool, err := pb.StartPool(pbs...)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-self.done:
|
||||
for _, p := range pbs {
|
||||
if !p.IsFinished() {
|
||||
p.Set(full)
|
||||
p.Finish()
|
||||
}
|
||||
}
|
||||
pool.Stop()
|
||||
return
|
||||
case <-time.After(time.Millisecond * 200):
|
||||
for i, w := range self.workers {
|
||||
if w.done {
|
||||
if !pbs[i].IsFinished() {
|
||||
pbs[i].Set(full)
|
||||
pbs[i].Finish()
|
||||
}
|
||||
} else {
|
||||
current := int(float64(full) * float64(w.downloaded) / float64(w.total))
|
||||
pbs[i].Set(current)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (self *DownloadTask) Download() {
|
||||
select {
|
||||
case <-self.done:
|
||||
panic(errors.New("[DownloadTask] already downloaded."))
|
||||
default:
|
||||
}
|
||||
|
||||
self.initWorkers()
|
||||
for _, w := range self.workers {
|
||||
go w.run()
|
||||
}
|
||||
|
||||
go self.showProgress()
|
||||
self.wait()
|
||||
self.merge()
|
||||
log.Printf("[DownloadTask] download %s done.", self.url)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package quickdown
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var client *http.Client
|
||||
|
||||
func Download(url string) error {
|
||||
return download(url, "", 5)
|
||||
}
|
||||
|
||||
func DownloadTo(url, filename string) error {
|
||||
return download(url, filename, 5)
|
||||
}
|
||||
|
||||
func DownloadWithWorkers(url string, workerCount int) error {
|
||||
return download(url, "", workerCount)
|
||||
}
|
||||
|
||||
func DownloadWithWorkersTo(url string, workerCount int, filename string) error {
|
||||
return download(url, filename, workerCount)
|
||||
}
|
||||
|
||||
func download(url, filename string, workerCount int) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
switch x := r.(type) {
|
||||
case string:
|
||||
err = errors.New(x)
|
||||
case error:
|
||||
err = x
|
||||
default:
|
||||
err = errors.New("Unknown panic")
|
||||
}
|
||||
log.Println(err)
|
||||
}
|
||||
}()
|
||||
NewDownloadTask(url, filename, workerCount).Download()
|
||||
return
|
||||
}
|
||||
|
||||
func SetHttpClient(c *http.Client) {
|
||||
client = c
|
||||
}
|
||||
|
||||
func GetHttpClient() (c *http.Client) {
|
||||
return client
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package quickdown
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type worker struct {
|
||||
id int
|
||||
url string
|
||||
filename string
|
||||
min, max int
|
||||
downloaded, total int
|
||||
done bool
|
||||
wg *sync.WaitGroup
|
||||
}
|
||||
|
||||
func newWorker(id int, url string, min, max int, wg *sync.WaitGroup, filename string) *worker {
|
||||
return &worker{
|
||||
id: id,
|
||||
url: url,
|
||||
min: min,
|
||||
max: max,
|
||||
total: max - min,
|
||||
wg: wg,
|
||||
filename: filename,
|
||||
}
|
||||
}
|
||||
|
||||
func (self *worker) run() {
|
||||
if self.done {
|
||||
panic(errors.New(fmt.Sprintf("worker(id=%d, url=%s) already done.", self.id, self.url)))
|
||||
}
|
||||
|
||||
var tmpfile *os.File
|
||||
fInfo, err := os.Stat(self.filename)
|
||||
if err == nil {
|
||||
fSize := int(fInfo.Size())
|
||||
self.min += fSize
|
||||
self.downloaded = fSize
|
||||
|
||||
if fSize >= self.total {
|
||||
self.setDone()
|
||||
return
|
||||
}
|
||||
|
||||
tmpfile, err = os.OpenFile(self.filename, os.O_APPEND|os.O_WRONLY, 0600)
|
||||
} else {
|
||||
tmpfile, _ = os.Create(self.filename)
|
||||
}
|
||||
defer tmpfile.Close()
|
||||
|
||||
request, _ := http.NewRequest("GET", self.url, nil)
|
||||
request.Header.Add("User-Agent", "github.com/xlzd/godown")
|
||||
if self.min >= 0 && self.max > 0 {
|
||||
request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", self.min, self.max))
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
for i := 0; ; i++ {
|
||||
buf := make([]byte, 4096*16)
|
||||
n, err := response.Body.Read(buf)
|
||||
if n != 0 {
|
||||
self.downloaded += n
|
||||
tmpfile.Write(buf[:n])
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
tmpfile.Sync()
|
||||
self.setDone()
|
||||
}
|
||||
|
||||
func (self *worker) setDone() {
|
||||
self.done = true
|
||||
self.wg.Done()
|
||||
}
|
||||
Reference in new issue
Block a user