|
|
@ -18,38 +18,24 @@ import (
|
|
|
|
"github.com/chromedp/chromedp/kb"
|
|
|
|
"github.com/chromedp/chromedp/kb"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
var startFlag = flag.String("start", "", "skip all the photos more recent than the one at that URL")
|
|
|
|
var (
|
|
|
|
|
|
|
|
nItemsFlag = flag.Int("n", -1, "number of items to download. If negative, get them all.")
|
|
|
|
func (s *Session) cleanDlDir() error {
|
|
|
|
devFlag = flag.Bool("dev", false, "dev mode. we reuse the same session dir (/tmp/gphotos-cdp), so we don't have to auth at every run.")
|
|
|
|
if s.dlDir == "" {
|
|
|
|
dlDirFlag = flag.String("dldir", "", "where to (temporarily) write the downloads. defaults to $HOME/Downloads/gphotos-cdp.")
|
|
|
|
return nil
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
entries, err := ioutil.ReadDir(s.dlDir)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, v := range entries {
|
|
|
|
|
|
|
|
if v.IsDir() {
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
func main() {
|
|
|
|
flag.Parse()
|
|
|
|
flag.Parse()
|
|
|
|
|
|
|
|
if *nItemsFlag == 0 {
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
}
|
|
|
|
s, err := NewSession()
|
|
|
|
s, err := NewSession()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
defer s.Shutdown()
|
|
|
|
defer s.Shutdown()
|
|
|
|
|
|
|
|
|
|
|
|
log.Printf("Dir: %v", s.tempDir)
|
|
|
|
log.Printf("Session Dir: %v", s.profileDir)
|
|
|
|
|
|
|
|
|
|
|
|
// s.fixPreferences()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if err := s.cleanDlDir(); err != nil {
|
|
|
|
if err := s.cleanDlDir(); err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
log.Fatal(err)
|
|
|
@ -67,8 +53,8 @@ func main() {
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
|
|
// chromedp.Sleep(30000*time.Millisecond),
|
|
|
|
chromedp.Sleep(30000*time.Millisecond),
|
|
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
|
|
// chromedp.Sleep(5000*time.Millisecond),
|
|
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
|
|
chromedp.ActionFunc(func(ctx context.Context) error {
|
|
|
|
log.Printf("post-navigate")
|
|
|
|
log.Printf("post-navigate")
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
@ -82,7 +68,6 @@ func main() {
|
|
|
|
log.Fatal(err)
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// download := func(ctx context.Context, dir string) (string, error) {
|
|
|
|
|
|
|
|
download := func(ctx context.Context) (string, error) {
|
|
|
|
download := func(ctx context.Context) (string, error) {
|
|
|
|
dir := s.dlDir
|
|
|
|
dir := s.dlDir
|
|
|
|
keyD, ok := kb.Keys['D']
|
|
|
|
keyD, ok := kb.Keys['D']
|
|
|
@ -112,15 +97,12 @@ func main() {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// TODO(mpl): use ioutil.Readdir.
|
|
|
|
|
|
|
|
// Also haha, there is a .DS_Store file
|
|
|
|
|
|
|
|
started := false
|
|
|
|
started := false
|
|
|
|
endTimeout := time.Now().Add(30 * time.Second)
|
|
|
|
endTimeout := time.Now().Add(30 * time.Second)
|
|
|
|
startTimeout := time.Now().Add(5 * time.Second)
|
|
|
|
startTimeout := time.Now().Add(5 * time.Second)
|
|
|
|
tick := 500 * time.Millisecond
|
|
|
|
tick := 500 * time.Millisecond
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
time.Sleep(tick)
|
|
|
|
time.Sleep(tick)
|
|
|
|
println("TICK")
|
|
|
|
|
|
|
|
if time.Now().After(endTimeout) {
|
|
|
|
if time.Now().After(endTimeout) {
|
|
|
|
return "", fmt.Errorf("timeout while downloading in %q", dir)
|
|
|
|
return "", fmt.Errorf("timeout while downloading in %q", dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -149,12 +131,8 @@ func main() {
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(fileEntries) > 1 {
|
|
|
|
if len(fileEntries) > 1 {
|
|
|
|
for _, v := range entries {
|
|
|
|
|
|
|
|
println(v)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), dir)
|
|
|
|
return "", fmt.Errorf("more than one file (%d) in download dir %q", len(fileEntries), dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
println(fileEntries[0])
|
|
|
|
|
|
|
|
if !strings.HasSuffix(fileEntries[0], ".crdownload") {
|
|
|
|
if !strings.HasSuffix(fileEntries[0], ".crdownload") {
|
|
|
|
// download is over
|
|
|
|
// download is over
|
|
|
|
return fileEntries[0], nil
|
|
|
|
return fileEntries[0], nil
|
|
|
@ -171,7 +149,6 @@ func main() {
|
|
|
|
if err := os.Rename(filepath.Join(s.dlDir, dlFile), filepath.Join(dir, dlFile)); err != nil {
|
|
|
|
if err := os.Rename(filepath.Join(s.dlDir, dlFile), filepath.Join(dir, dlFile)); err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
println("NEW FILE: ", filepath.Join(dir, dlFile))
|
|
|
|
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -185,8 +162,6 @@ func main() {
|
|
|
|
return mvDl(dlFile)(ctx)
|
|
|
|
return mvDl(dlFile)(ctx)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// firstNav := func(ctx context.Context) chromedp.ActionFunc {
|
|
|
|
|
|
|
|
// return func(ctx context.Context) error {
|
|
|
|
|
|
|
|
firstNav := func(ctx context.Context) error {
|
|
|
|
firstNav := func(ctx context.Context) error {
|
|
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
|
|
log.Printf("sent key")
|
|
|
|
log.Printf("sent key")
|
|
|
@ -195,7 +170,6 @@ func main() {
|
|
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
navRight := func(ctx context.Context) error {
|
|
|
|
navRight := func(ctx context.Context) error {
|
|
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
|
@ -205,21 +179,21 @@ func main() {
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
navRightN := func(N int) func(context.Context) error {
|
|
|
|
navRightN := func(N int, ctx context.Context) chromedp.ActionFunc {
|
|
|
|
|
|
|
|
n := 0
|
|
|
|
n := 0
|
|
|
|
return func(ctx context.Context) error {
|
|
|
|
return func(ctx context.Context) error {
|
|
|
|
|
|
|
|
if N == 0 {
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
if n >= N {
|
|
|
|
if N > 0 && n >= N {
|
|
|
|
break
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
chromedp.KeyEvent(kb.ArrowRight).Do(ctx)
|
|
|
|
|
|
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
|
|
|
|
|
|
if err := navRight(ctx); err != nil {
|
|
|
|
if err := navRight(ctx); err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
chromedp.Sleep(500 * time.Millisecond).Do(ctx)
|
|
|
|
// TODO(mpl): deal with getting the very last photo to properly exit that loop when N < 0.
|
|
|
|
if err := download.Do(ctx); err != nil {
|
|
|
|
if err := dlAndMove(ctx); err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
n++
|
|
|
|
n++
|
|
|
@ -227,13 +201,10 @@ func main() {
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_, _, _ = download, navRight, firstNav
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if err := chromedp.Run(ctx,
|
|
|
|
if err := chromedp.Run(ctx,
|
|
|
|
page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
|
|
|
|
page.SetDownloadBehavior(page.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(s.dlDir),
|
|
|
|
// TODO(mpl): add policy func over photo URL, which decides what we do (with?)
|
|
|
|
// TODO(mpl): add policy func over photo URL, which decides what we do with the downloaded file. default policy is storing it on disk.
|
|
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
|
|
chromedp.Navigate("https://photos.google.com/"),
|
|
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
|
|
chromedp.Sleep(5000*time.Millisecond),
|
|
|
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
|
|
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
|
|
@ -243,8 +214,7 @@ func main() {
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
chromedp.ActionFunc(firstNav),
|
|
|
|
chromedp.ActionFunc(firstNav),
|
|
|
|
chromedp.ActionFunc(dlAndMove),
|
|
|
|
chromedp.ActionFunc(dlAndMove),
|
|
|
|
chromedp.ActionFunc(navRight),
|
|
|
|
chromedp.ActionFunc(navRightN(*nItemsFlag-1)),
|
|
|
|
chromedp.ActionFunc(dlAndMove),
|
|
|
|
|
|
|
|
); err != nil {
|
|
|
|
); err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -260,24 +230,34 @@ type Session struct {
|
|
|
|
parentContext context.Context
|
|
|
|
parentContext context.Context
|
|
|
|
parentCancel context.CancelFunc
|
|
|
|
parentCancel context.CancelFunc
|
|
|
|
dlDir string
|
|
|
|
dlDir string
|
|
|
|
tempDir string
|
|
|
|
profileDir string
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func NewSession() (*Session, error) {
|
|
|
|
func NewSession() (*Session, error) {
|
|
|
|
/*
|
|
|
|
var dir string
|
|
|
|
dir, err := ioutil.TempDir("", "footest")
|
|
|
|
if *devFlag {
|
|
|
|
|
|
|
|
dir = filepath.Join(os.TempDir(), "gphotos-cdp")
|
|
|
|
|
|
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
dir, err = ioutil.TempDir("", "gphotos-cdp")
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*/
|
|
|
|
|
|
|
|
dir := "/Users/mpl/chromedp"
|
|
|
|
|
|
|
|
s := &Session{
|
|
|
|
|
|
|
|
tempDir: dir,
|
|
|
|
|
|
|
|
dlDir: filepath.Join(os.Getenv("HOME"), "Downloads", "pk-gphotos"),
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := os.MkdirAll(s.dlDir, 0700); err != nil {
|
|
|
|
dlDir := *dlDirFlag
|
|
|
|
|
|
|
|
if dlDir == "" {
|
|
|
|
|
|
|
|
dlDir = filepath.Join(os.Getenv("HOME"), "Downloads", "gphotos-cdp")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.MkdirAll(dlDir, 0700); err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
s := &Session{
|
|
|
|
|
|
|
|
profileDir: dir,
|
|
|
|
|
|
|
|
dlDir: dlDir,
|
|
|
|
|
|
|
|
}
|
|
|
|
return s, nil
|
|
|
|
return s, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -285,7 +265,7 @@ func (s *Session) NewContext() (context.Context, context.CancelFunc) {
|
|
|
|
ctx, cancel := chromedp.NewExecAllocator(context.Background(),
|
|
|
|
ctx, cancel := chromedp.NewExecAllocator(context.Background(),
|
|
|
|
chromedp.NoFirstRun,
|
|
|
|
chromedp.NoFirstRun,
|
|
|
|
chromedp.NoDefaultBrowserCheck,
|
|
|
|
chromedp.NoDefaultBrowserCheck,
|
|
|
|
chromedp.UserDataDir(s.tempDir),
|
|
|
|
chromedp.UserDataDir(s.profileDir),
|
|
|
|
|
|
|
|
|
|
|
|
chromedp.Flag("disable-background-networking", true),
|
|
|
|
chromedp.Flag("disable-background-networking", true),
|
|
|
|
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
|
|
|
|
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
|
|
|
@ -319,3 +299,22 @@ func (s *Session) NewContext() (context.Context, context.CancelFunc) {
|
|
|
|
func (s *Session) Shutdown() {
|
|
|
|
func (s *Session) Shutdown() {
|
|
|
|
s.parentCancel()
|
|
|
|
s.parentCancel()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func (s *Session) cleanDlDir() error {
|
|
|
|
|
|
|
|
if s.dlDir == "" {
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
entries, err := ioutil.ReadDir(s.dlDir)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, v := range entries {
|
|
|
|
|
|
|
|
if v.IsDir() {
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.Remove(filepath.Join(s.dlDir, v.Name())); err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|