Merge pull request #580 from FiloSottile/master

The new shiny build system

.gitignore

@@ -4,3 +4,12 @@
 wine-py2exe/
 py2exe.log
 *.kate-swp
+build/
+dist/
+MANIFEST
+README.txt
+youtube-dl.1
+youtube-dl.bash-completion
+youtube-dl
+youtube-dl.exe
+youtube-dl.tar.gz

.travis.yml

@@ -1,11 +1,13 @@
 language: python
-#specify the python version
 python:
   - "2.6"
   - "2.7"
-notifications:
+#  - "3.3"
-  irc: "irc.freenode.org#youtube-dl"
-#command to install the setup
-install:
-# command to run tests
 script: nosetests test --verbose
+notifications:
+  email:
+    - filippo.valsorda@gmail.com
+  irc:
+    channels:
+      - "irc.freenode.org#youtube-dl"
+    skip_join: true

LATEST_VERSION

@@ -1 +1 @@
-2012.11.29
+9999.99.99

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include test/*.py
+include test/*.json

Makefile

@@ -1,8 +1,7 @@
-all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
-# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
 
 clean:
-	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
+	rm -rf youtube-dl youtube-dl.exe youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
@@ -20,8 +19,7 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 test:
 	nosetests2 --nocapture test
 
-.PHONY: all clean install test README.md youtube-dl.bash-completion
+.PHONY: all clean install test
-# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
 
 youtube-dl: youtube_dl/*.py
 	zip --quiet youtube-dl youtube_dl/*.py
@@ -31,28 +29,20 @@ youtube-dl: youtube_dl/*.py
 	rm youtube-dl.zip
 	chmod a+x youtube-dl
 
-youtube-dl.exe: youtube_dl/*.py
-	bash devscripts/wine-py2exe.sh build_exe.py
-
 README.md: youtube_dl/*.py
-	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
+	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
-		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
+
-		footer=$$(sed -e '1,/.*# CONFIGURATION/ d' README.md) && \
+README.txt: README.md
-		echo "$${header}" > README.md && \
+	pandoc -f markdown -t plain README.md -o README.txt
-		echo >> README.md && \
-		echo '# OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '# CONFIGURATION' >> README.md && \
-		echo "$${footer}" >> README.md
 
 youtube-dl.1: README.md
-	pandoc -s -w man README.md -o youtube-dl.1
+	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
-youtube-dl.bash-completion: README.md
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.template
-	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
+	python devscripts/bash-completion.py
-		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
-		echo "$${content}" > youtube-dl.bash-completion
 
-LATEST_VERSION: youtube_dl/__init__.py
+youtube-dl.tar.gz: all
-	python -m youtube_dl --version > LATEST_VERSION
+	tar -czf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" \
+		--exclude="*.pyc" --exclude="*.pyo" --exclude="*~" --exclude="youtube-dl.exe" \
+		--exclude="wine-py2exe/" --exclude="py2exe.log" --exclude="*.kate-swp" \
+		--exclude="build/" --exclude="dist/" --exclude="MANIFEST" --exclude=".git/" .

README.md

@@ -51,7 +51,10 @@ ## Filesystem Options:
                              %(upload_date)s for the upload date (YYYYMMDD),
                              %(extractor)s for the provider (youtube, metacafe,
                              etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout.
+                             percent. Use - to output to stdout. Can also be
+                             used to download to a different directory, for
+                             example with -o '/my/downloads/%(uploader)s/%(title
+                             )s-%(id)s.%(ext)s' .
     --restrict-filenames     Restrict filenames to only ASCII characters, and
                              avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)

bin/youtube-dl

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import youtube_dl
+
+if __name__ == '__main__':
+    youtube_dl.main()

build_exe.py

@@ -1,48 +0,0 @@
-from distutils.core import setup
-import py2exe
-import sys, os
-
-"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
-
-# If run without args, build executables
-if len(sys.argv) == 1:
-    sys.argv.append("py2exe")
-
-# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
-sys.path.append('./youtube_dl')
-
-options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-console = [{
-    "script":"./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
-}]
-
-init_file = open('./youtube_dl/__init__.py')
-for line in init_file.readlines():
-    if line.startswith('__version__'):
-        version = line[11:].strip(" ='\n")
-        break
-else:
-    version = ''
-
-setup(name='youtube-dl',
-      version=version,
-      description='Small command-line program to download videos from YouTube.com and other video sites',
-      url='https://github.com/rg3/youtube-dl',
-      packages=['youtube_dl'],
-      
-      console = console,
-      options = {"py2exe": options},
-      zipfile = None,
-)
-
-import shutil
-shutil.rmtree("build")
-

devscripts/bash-completion.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.template"
+
+def build_completion(opt_parser):
+    opts_flag = []
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            #for every long flag
+            opts_flag.append(option.get_opt_string())
+    with open(BASH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    with open(BASH_COMPLETION_FILE, "w") as f:
+        #just using the special char
+        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

devscripts/bash-completion.template

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="{{flags}}"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

devscripts/make_readme.py

@@ -0,0 +1,20 @@
+import sys
+import re
+
+README_FILE = 'README.md'
+helptext = sys.stdin.read()
+
+with open(README_FILE) as f:
+    oldreadme = f.read()
+
+header = oldreadme[:oldreadme.index('# OPTIONS')]
+footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+
+options = helptext[helptext.index('  General Options:')+19:]
+options = re.sub(r'^  (\w.+)$', r'## \1', options, flags=re.M)
+options = '# OPTIONS\n' + options + '\n'
+
+with open(README_FILE, 'w') as f:
+    f.write(header)
+    f.write(options)
+    f.write(footer)

devscripts/transition_helper.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+try:
+	raw_input()
+except NameError: # Python 3
+	input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+try:
+    urlh = compat_urllib_request.urlopen(BIN_URL)
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    with open(filename, 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

devscripts/transition_helper_exe/setup.py

@@ -0,0 +1,12 @@
+from distutils.core import setup
+import py2exe
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)

devscripts/transition_helper_exe/youtube-dl.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+import sys, os
+import urllib2
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+raw_input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+exe = os.path.abspath(filename)
+directory = os.path.dirname(exe)
+if not os.access(directory, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % directory)
+
+try:
+    urlh = urllib2.urlopen(EXE_URL)
+    newcontent = urlh.read()
+    urlh.close()
+    with open(exe + '.new', 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    bat = os.path.join(directory, 'youtube-dl-updater.bat')
+    b = open(bat, 'w')
+    b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+    \n""" %(exe, exe, bat))
+    b.close()
+
+    os.startfile(bat)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

setup.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+from distutils.core import setup
+import pkg_resources
+import sys
+
+try:
+    import py2exe
+    """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+except ImportError:
+    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+        print("Cannot import py2exe", file=sys.stderr)
+        exit(1)
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+py2exe_console = [{
+    "script": "./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+py2exe_params = {
+    'console': py2exe_console,
+    'options': { "py2exe": py2exe_options },
+    'zipfile': None
+}
+
+if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+    params = py2exe_params
+else:
+    params = {
+        'scripts': ['bin/youtube-dl'],
+        'data_files': [('etc/bash_completion.d', ['youtube-dl.bash-completion']), # Installing system-wide would require sudo...
+                       ('share/doc/youtube_dl', ['README.txt']),
+                       ('share/man/man1/', ['youtube-dl.1'])]
+    }
+
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec'))
+
+setup(
+    name = 'youtube_dl',
+    version = __version__,
+    description = 'YouTube video downloader',
+    long_description = 'Small command-line program to download videos from YouTube.com and other video sites.',
+    url = 'https://github.com/rg3/youtube-dl',
+    author = 'Ricardo Garcia',
+    maintainer = 'Philipp Hagemeister',
+    maintainer_email = 'phihag@phihag.de',
+    packages = ['youtube_dl'],
+
+    # Provokes warning on most systems (why?!)
+    #test_suite = 'nose.collector',
+    #test_requires = ['nosetest'],
+
+    classifiers = [
+        "Topic :: Multimedia :: Video",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "License :: Public Domain",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3"
+    ],
+
+    **params
+)

youtube-dl.1

@@ -1,306 +0,0 @@
-.TH YOUTUBE-DL 1 "" 
-.SH NAME
-.PP
-youtube-dl
-.SH SYNOPSIS
-.PP
-\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
-.SH DESCRIPTION
-.PP
-\f[B]youtube-dl\f[] is a small command-line program to download videos
-from YouTube.com and a few more sites.
-It requires the Python interpreter, version 2.x (x being at least 6),
-and it is not platform specific.
-It should work in your Unix box, in Windows or in Mac OS X.
-It is released to the public domain, which means you can modify it,
-redistribute it or use it however you like.
-.SH OPTIONS
-.IP
-.nf
-\f[C]
--h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
---version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
--U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
--i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
--r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
--R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
---buffer-size\ SIZE\ \ \ \ \ \ \ size\ of\ download\ buffer\ (e.g.\ 1024\ or\ 16k)\ (default
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ is\ 1024)
---no-resize-buffer\ \ \ \ \ \ \ do\ not\ automatically\ adjust\ the\ buffer\ size.\ By
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ the\ buffer\ size\ is\ automatically\ resized
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ an\ initial\ value\ of\ SIZE.
---dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
---user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
---list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
-\f[]
-.fi
-.SS Video Selection:
-.IP
-.nf
-\f[C]
---playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
---playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
---match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
---reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
---max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
-\f[]
-.fi
-.SS Filesystem Options:
-.IP
-.nf
-\f[C]
--t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
---id\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ video\ ID\ in\ file\ name
--l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ [deprecated]\ alias\ of\ --title
--A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
--o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(title)s\ to\ get\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
---restrict-filenames\ \ \ \ \ Restrict\ filenames\ to\ only\ ASCII\ characters,\ and
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avoid\ "&"\ and\ spaces\ in\ filenames
--a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
--w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
--c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
---no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
---cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
---no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
---no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
---write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
---write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
-\f[]
-.fi
-.SS Verbosity / Simulation Options:
-.IP
-.nf
-\f[C]
--q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
--s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
---skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
--g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
--e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
---get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
---get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
---get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
---get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
---no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
---console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
--v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
-\f[]
-.fi
-.SS Video Format Options:
-.IP
-.nf
-\f[C]
--f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
---all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
---prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
---max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
--F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
---write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
---srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
-\f[]
-.fi
-.SS Authentication Options:
-.IP
-.nf
-\f[C]
--u,\ --username\ USERNAME\ \ account\ username
--p,\ --password\ PASSWORD\ \ account\ password
--n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
-\f[]
-.fi
-.SS Post-processing Options:
-.IP
-.nf
-\f[C]
--x,\ --extract-audio\ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
---audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
---audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
--k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
-\f[]
-.fi
-.SH CONFIGURATION
-.PP
-You can configure youtube-dl by placing default arguments (such as
-\f[C]--extract-audio\ --no-mtime\f[] to always extract the audio and not
-copy the mtime) into \f[C]/etc/youtube-dl.conf\f[] and/or
-\f[C]~/.local/config/youtube-dl.conf\f[].
-.SH OUTPUT TEMPLATE
-.PP
-The \f[C]-o\f[] option allows users to indicate a template for the
-output file names.
-The basic usage is not to set any template arguments when downloading a
-single file, like in
-\f[C]youtube-dl\ -o\ funny_video.flv\ "http://some/video"\f[].
-However, it may contain special sequences that will be replaced when
-downloading each video.
-The special sequences have the format \f[C]%(NAME)s\f[].
-To clarify, that is a percent symbol followed by a name in parenthesis,
-followed by a lowercase S.
-Allowed names are:
-.IP \[bu] 2
-\f[C]id\f[]: The sequence will be replaced by the video identifier.
-.IP \[bu] 2
-\f[C]url\f[]: The sequence will be replaced by the video URL.
-.IP \[bu] 2
-\f[C]uploader\f[]: The sequence will be replaced by the nickname of the
-person who uploaded the video.
-.IP \[bu] 2
-\f[C]upload_date\f[]: The sequence will be replaced by the upload date
-in YYYYMMDD format.
-.IP \[bu] 2
-\f[C]title\f[]: The sequence will be replaced by the video title.
-.IP \[bu] 2
-\f[C]ext\f[]: The sequence will be replaced by the appropriate extension
-(like flv or mp4).
-.IP \[bu] 2
-\f[C]epoch\f[]: The sequence will be replaced by the Unix epoch when
-creating the file.
-.IP \[bu] 2
-\f[C]autonumber\f[]: The sequence will be replaced by a five-digit
-number that will be increased with each download, starting at zero.
-.PP
-The current default template is \f[C]%(id)s.%(ext)s\f[], but that will
-be switchted to \f[C]%(title)s-%(id)s.%(ext)s\f[] (which can be
-requested with \f[C]-t\f[] at the moment).
-.PP
-In some cases, you don\[aq]t want special characters such as 中, spaces,
-or &, such as when transferring the downloaded filename to a Windows
-system or the filename through an 8bit-unsafe channel.
-In these cases, add the \f[C]--restrict-filenames\f[] flag to get a
-shorter title:
-.IP
-.nf
-\f[C]
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc
-youtube-dl\ test\ video\ \[aq]\[aq]_ä↭𝕐.mp4\ \ \ \ #\ All\ kinds\ of\ weird\ characters
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc\ --restrict-filenames
-youtube-dl_test_video_.mp4\ \ \ \ \ \ \ \ \ \ #\ A\ simple\ file\ name
-\f[]
-.fi
-.SH FAQ
-.SS Can you please put the -b option back?
-.PP
-Most people asking this question are not aware that youtube-dl now
-defaults to downloading the highest available quality as reported by
-YouTube, which will be 1080p or 720p in some cases, so you no longer
-need the -b option.
-For some specific videos, maybe YouTube does not report them to be
-available in a specific high quality format you\[aq]\[aq]re interested
-in.
-In that case, simply request it with the -f option and youtube-dl will
-try to download it.
-.SS I get HTTP error 402 when trying to download a video. What\[aq]s
-this?
-.PP
-Apparently YouTube requires you to pass a CAPTCHA test if you download
-too much.
-We\[aq]\[aq]re considering to provide a way to let you solve the
-CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
-moment, your best course of action is pointing a webbrowser to the
-youtube URL, solving the CAPTCHA, and restart youtube-dl.
-.SS I have downloaded a video but how can I play it?
-.PP
-Once the video is fully downloaded, use any video player, such as
-vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
-.SS The links provided by youtube-dl -g are not working anymore
-.PP
-The URLs youtube-dl outputs require the downloader to have the correct
-cookies.
-Use the \f[C]--cookies\f[] option to write the required cookies into a
-file, and advise your downloader to read cookies from that file.
-Some sites also require a common user agent to be used, use
-\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
-.SS ERROR: no fmt_url_map or conn information found in video info
-.PP
-youtube has switched to a new video info format in July 2011 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS ERROR: unable to download video
-.PP
-youtube requires an additional signature since September 2012 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS SyntaxError: Non-ASCII character
-.PP
-The error
-.IP
-.nf
-\f[C]
-File\ "youtube-dl",\ line\ 2
-SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
-\f[]
-.fi
-.PP
-means you\[aq]re using an outdated version of Python.
-Please update to Python 2.6 or 2.7.
-.PP
-To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
-out like this:
-.IP
-.nf
-\f[C]
-git\ clone\ git://github.com/rg3/youtube-dl.git
-cd\ youtube-dl
-python\ -m\ youtube_dl\ --help
-\f[]
-.fi
-.PP
-Please note that Python 2.5 is not supported anymore.
-.SS What is this binary file? Where has the code gone?
-.PP
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
-simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
-some systems) or clone the git repository, as laid out above.
-If you modify the code, you can run it by executing the
-\f[C]__main__.py\f[] file.
-To recompile the executable, run \f[C]make\ youtube-dl\f[].
-.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
-.PP
-To run the exe you need to install first the Microsoft Visual C++ 2008
-Redistributable
-Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
-.SH COPYRIGHT
-.PP
-youtube-dl is released into the public domain by the copyright holders.
-.PP
-This README file was originally written by Daniel Bolton
-(<https://github.com/dbbolton>) and is likewise released into the public
-domain.
-.SH BUGS
-.PP
-Bugs and suggestions should be reported at:
-<https://github.com/rg3/youtube-dl/issues>
-.PP
-Please include:
-.IP \[bu] 2
-Your exact command line, like
-\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
-A common mistake is not to escape the \f[C]&\f[].
-Putting URLs in quotes should solve this problem.
-.IP \[bu] 2
-The output of \f[C]youtube-dl\ --version\f[]
-.IP \[bu] 2
-The output of \f[C]python\ --version\f[]
-.IP \[bu] 2
-The name and version of your Operating System ("Ubuntu 11.04 x64" or
-"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -1,14 +0,0 @@
-__youtube-dl()
-{
-    local cur prev opts
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --buffer-size --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --no-resize-buffer --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --restrict-filenames --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
-
-    if [[ ${cur} == * ]] ; then
-        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        return 0
-    fi
-}
-
-complete -F __youtube-dl youtube-dl

youtube_dl/__init__.py

@@ -23,12 +23,6 @@
     )
 
 __license__ = 'Public Domain'
-__version__ = '2012.11.29'
-
-UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
-UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
-UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
-
 
 import getpass
 import optparse
@@ -41,34 +35,46 @@
 import warnings
 
 from .utils import *
+from .version import __version__
 from .FileDownloader import *
 from .InfoExtractors import *
 from .PostProcessor import *
 
 def updateSelf(downloader, filename):
-    ''' Update the program file with the latest version from the repository '''
+    """Update the program file with the latest version from the repository"""
-    # Note: downloader only used for options
 
-    if not os.access(filename, os.W_OK):
+    # TODO: at least, check https certificates
-        sys.exit('ERROR: no write permissions on %s' % filename)
 
-    downloader.to_screen(u'Updating to latest version...')
+    from zipimport import zipimporter
 
-    urlv = compat_urllib_request.urlopen(UPDATE_URL_VERSION)
+    API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
-    newversion = urlv.read().strip()
+    BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
-    if newversion == __version__:
+    EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe"
-        downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+
-        return
+    if hasattr(sys, "frozen"): # PY2EXE
-    urlv.close()
+        if not os.access(filename, os.W_OK):
+            sys.exit('ERROR: no write permissions on %s' % filename)
+
+        downloader.to_screen(u'Updating to latest version...')
+
+        urla = compat_urllib_request.urlopen(API_URL)
+        download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read()))
+        if not download:
+            downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+            return
+        newversion = download[0]["description"].strip()
+        if newversion == __version__:
+            downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+            return
+        urla.close()
 
-    if hasattr(sys, "frozen"): #py2exe
         exe = os.path.abspath(filename)
         directory = os.path.dirname(exe)
         if not os.access(directory, os.W_OK):
             sys.exit('ERROR: no write permissions on %s' % directory)
 
         try:
-            urlh = compat_urllib_request.urlopen(UPDATE_URL_EXE)
+            urlh = compat_urllib_request.urlopen(EXE_URL)
             newcontent = urlh.read()
             urlh.close()
             with open(exe + '.new', 'wb') as outf:
@@ -91,9 +97,25 @@ def updateSelf(downloader, filename):
         except (IOError, OSError) as err:
             sys.exit('ERROR: unable to overwrite current version')
 
-    else:
+    elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP
+        if not os.access(filename, os.W_OK):
+            sys.exit('ERROR: no write permissions on %s' % filename)
+
+        downloader.to_screen(u'Updating to latest version...')
+
+        urla = compat_urllib_request.urlopen(API_URL)
+        download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"]
+        if not download:
+            downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+            return
+        newversion = download[0]["description"].strip()
+        if newversion == __version__:
+            downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+            return
+        urla.close()
+
         try:
-            urlh = compat_urllib_request.urlopen(UPDATE_URL)
+            urlh = compat_urllib_request.urlopen(BIN_URL)
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError) as err:
@@ -105,6 +127,10 @@ def updateSelf(downloader, filename):
         except (IOError, OSError) as err:
             sys.exit('ERROR: unable to overwrite current version')
 
+    else:
+        downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.')
+        return
+
     downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
 
 def parseOpts():

youtube_dl/__main__.py

@@ -6,7 +6,8 @@
 
 import sys
 
-if __package__ is None:
+if __package__ is None and not hasattr(sys, "frozen"):
+    # direct call of __main__.py
     import os.path
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 

youtube_dl/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '2012.11.29'