package main
import ( "context" "encoding/json" "fmt" "io/ioutil" "log" "net/http" "net/url" "os" "path" "runtime" "strconv" "strings" "sync" "time"
"github.com/schollz/progressbar"
"github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" "github.com/chromedp/chromedp/kb" )
const ( homeUrl string = "https://manhuabika.com/pLogin/" pageUrl string = "https://manhuabika.com/pcomicview/?cid=" pageUrl2 string = "https://manhuabika.com/pchapter/?cid=" )
var ( account string password string idList []string savePath string PROXY string ctx context.Context wg sync.WaitGroup NmList [100][100]string picChan chan *pic )
type pic struct { bkNum int chNum int picNum int src string }
func main() { if len(idList) == 0 { fmt.Println("no books") return } Login()
wg.Add(1) go PicsDownload()
for index, id := range idList { LoadBook(index, id) } close(picChan)
wg.Wait() }
func standarlizeName(oriStr string) string { replacer := strings.NewReplacer("/", "", "|", "", "\\", "", ":", " ", "*", "", "?", "", "<", "[", ">", "]", ":", " ") return replacer.Replace(oriStr) }
func PicsDownload() { defer wg.Done() picChan = make(chan *pic, 100000) for picture := range picChan { wg.Add(1) go PicDownload(picture) } }
func PicDownload(picture *pic) { defer wg.Done() proxyAddress, _ := url.Parse(PROXY) client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyAddress)}}
req, _ := http.NewRequest("GET", picture.src, nil) resp, err := client.Do(req) Check(err)
body, err := ioutil.ReadAll(resp.Body) Check(err) defer resp.Body.Close()
filePath := NmList[picture.bkNum][picture.chNum] + "/" + fmt.Sprintf("%02d", picture.picNum) + ".jpg"
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0644) Check(err) defer f.Close()
f.Write(body) }
func Load(url string) { err := chromedp.Run(ctx, chromedp.Navigate(url), chromedp.WaitVisible(`//div[@class='appBottomMenu']`)) Check(err) time.Sleep(2 * time.Second) }
func LoadBook(index int, id string) { bookNm := "" var contentNode []*cdp.Node
Load(pageUrl + id) err := chromedp.Run(ctx, chromedp.TextContent(`//div[@class='comic-title text-start']`, &bookNm, chromedp.BySearch), chromedp.Nodes(`//div[@class="col-3 m-0 mb-1"]`, &contentNode)) Check(err)
fmt.Printf("\nStart to process book [%d]%s \n", index+1, bookNm)
NmList[index][0] = savePath + "/" + standarlizeName(bookNm)
chaps := len(contentNode) fmt.Printf("\nTotal: %d Chapters \n", chaps)
for i := 1; i <= chaps; i++ { chapNm := "" chromedp.Run(ctx, chromedp.TextContent(contentNode[chaps-i].PartialXPath(), &chapNm)) chapNm = standarlizeName(chapNm) fmt.Printf("[%d] %s \n", i, chapNm) NmList[index][i] = NmList[index][0] + "/[" + fmt.Sprintf("%02d", i) + "] " + chapNm
if !pathExists(NmList[index][i]) { err = os.MkdirAll(NmList[index][i], 0766) Check(err) } } fmt.Printf("\n") bar := progressbar.New(chaps) for i := 1; i <= chaps; i++ { LoadChap(index, i, bar) } }
func LoadChap(bookNum int, chapNum int, bar *progressbar.ProgressBar) { Load(pageUrl2 + idList[bookNum] + "&chapter=" + strconv.Itoa(chapNum)) ScrollDown()
var picNodes []*cdp.Node err := chromedp.Run(ctx, chromedp.Nodes(`//div[@class='chapter-images wide-block pt-2 pb-2 my-bg-white']/img`, &picNodes)) Check(err)
for i := 0; i < len(picNodes); i++ { picture := &pic{bookNum, chapNum, i, picNodes[i].AttributeValue("data-src")} picChan <- picture }
bar.Add(1) }
func ScrollDown() { var prgsBar string for i := 0; ; { err := chromedp.Run(ctx, chromedp.Evaluate("window.scrollBy(0,document.body.scrollHeight)", nil), chromedp.Sleep(2*time.Second), chromedp.TextContent(`.//div[@class="w-100 text-center text-black-50 my-read-tip"]`, &prgsBar)) Check(err) if prgsBar == "100 % (點擊可跳轉)" { if i > 2 { break } else { i++ } } else { if i != 0 { i = 0 } } } }
func Login() { err := chromedp.Run(ctx, chromedp.Navigate(homeUrl), chromedp.WaitVisible("#email1", chromedp.ByID), chromedp.SendKeys("#email1", account+kb.Tab+password, chromedp.ByID), chromedp.Click(`.//button[@type="submit"]`, chromedp.BySearch), chromedp.WaitVisible(`//div[@class='appBottomMenu']`)) Check(err) fmt.Println("login complete")
fmt.Println("\n...you've 15 second to change the picture quality...") time.Sleep(100) Load(pageUrl2 + idList[0] + "&chapter=1") time.Sleep(10 * time.Second) fmt.Println("processing start")
}
func pathExists(path string) bool { _, err := os.Stat(path) if err == nil { return true } if os.IsNotExist(err) { return false } return false }
func init() { headlessFlag := chromedp.Flag("headless", false) opts := append( chromedp.DefaultExecAllocatorOptions[:], chromedp.NoDefaultBrowserCheck, headlessFlag, chromedp.IgnoreCertErrors, chromedp.Flag("blink-settings", "imagesEnabled=false"), chromedp.DisableGPU, chromedp.NoSandbox, chromedp.NoFirstRun, chromedp.Flag("disable-web-security", true), chromedp.Flag("disable-extensions", true), chromedp.Flag("disable-default-apps", true), chromedp.WindowSize(1280, 1024), chromedp.Flag("run-all-compositor-stages-before-draw", true), chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`), )
allocCtx, _ := chromedp.NewExecAllocator(context.Background(), opts...) ctx, _ = chromedp.NewContext( allocCtx, chromedp.WithLogf(log.Printf), ) chromedp.Run(ctx, make([]chromedp.Action, 0, 1)...) fmt.Println("-- chromedp start success --") }
func init() { type custom struct { Account string `json:"Account"` Password string `json:"Password"` IDList []string `json:"IDList"` SavePath string `json:"SavePath"` PROXY string `json:"PROXY"` }
_, projectPath, _, _ := runtime.Caller(0) filePath := path.Dir(projectPath) + "/custom.json"
fmt.Println(">fetch customInfo from", filePath)
jsonFile, err := os.Open(filePath) Check(err) defer jsonFile.Close()
jsonData, err := ioutil.ReadAll(jsonFile) Check(err)
var customInfo custom json.Unmarshal(jsonData, &customInfo) account, password, idList, savePath, PROXY = customInfo.Account, customInfo.Password, customInfo.IDList, customInfo.SavePath, customInfo.PROXY
if PROXY == "" { PROXY = "http://127.0.0.1:1080/" }
fmt.Println("-- custom.json loadng success --") fmt.Println("Welcome!", account) fmt.Println("your savePath:", savePath) fmt.Println("your cidList:", idList) fmt.Println("your proxy:", idList) }
func Check(err error) { if err != nil { log.Fatal(err) } }
|