autogen: finisned windows readme generator

tools/readme_autogen/readme_autogen.py

@@ -3,7 +3,9 @@
 import json
 import os.path
 from string import Template
-from pprint import pprint
+from enum import IntEnum
+from itertools import groupby
+from functools import lru_cache
 
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 TEMPLATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
@@ -12,19 +14,47 @@ DATAFILE_PATH = os.path.join(BASE_PATH,
                              "..", "..", "drivers.json")
 LINUX_README_PATH = os.path.join(BASE_PATH,
                                  "..", "..", "README.md")
+WINDOWS_README_PATH = os.path.join(BASE_PATH,
+                                 "..", "..", "win", "README.md")
 ENCODING="utf-8"
 
-def template(filename):
+class Product(IntEnum):
+    GeForce = 10
+    Quadro = 20
+
+class WinSeries(IntEnum):
+    win10 = 10
+    win7 = 20
+    ws2012 = 30
+    ws2016 = 40
+
+PRODUCT_LABELS = {
+    Product.GeForce: "GeForce",
+    Product.Quadro: "Quadro",
+}
+
+WIN_SERIES_LABELS = {
+    WinSeries.win10: "Windows 10",
+    WinSeries.win7: "Windows 7, Windows 8, Windows 8.1",
+    WinSeries.ws2012: "Windows Server 2008R2, 2012, 2012R2",
+    WinSeries.ws2016: "Windows Server 2016, 2019",
+}
+
+@lru_cache(maxsize=None)
+def template(filename, strip_newlines=False):
     filename = os.path.join(TEMPLATE_PATH, filename)
     with open(filename, encoding=ENCODING) as f:
-        t = Template(f.read())
+        text = f.read()
+        if strip_newlines:
+            text = text.rstrip('\r\n')
+        t = Template(text)
     return t
 
 def version_key_fun(ver):
     return tuple(map(int, ver.split('.')))
 
 def find_driver(drivers, version, low=0, hi=None):
-    """ Bisect search on sorted drivers list """
+    """ Bisect search on sorted linux drivers list """
     if hi is None:
         hi = len(drivers)
     L = hi - low
@@ -43,17 +73,17 @@ def find_driver(drivers, version, low=0, hi=None):
             return drivers[low + L // 2]
 
 def linux_readme(data):
-    master_tmpl = template("linux_readme_master.tmpl")
-    linux_nolink_row_tmpl = template('linux_nolink_row.tmpl')
-    linux_link_row_tmpl = template('linux_link_row.tmpl')
+    master_tmpl = template('linux_readme_master.tmpl')
+    nolink_row_tmpl = template('linux_nolink_row.tmpl', True)
+    link_row_tmpl = template('linux_link_row.tmpl', True)
     drivers = sorted(data['drivers'],
                      key=lambda d: version_key_fun(d['version']))
     def row_gen():
         for drv in drivers:
             driver_url = drv.get('driver_url')
-            t = linux_nolink_row_tmpl if driver_url is None else linux_link_row_tmpl
+            t = nolink_row_tmpl if driver_url is None else link_row_tmpl
             yield t.substitute(driver_version=drv['version'],
-                               driver_url=driver_url).rstrip('\r\n')
+                               driver_url=driver_url)
     version_list = "\n".join(row_gen())
     latest_version = drivers[-1]['version']
     example_driver = find_driver(drivers, data['example']['version'])
@@ -64,12 +94,77 @@ def linux_readme(data):
                                   example_driver_version=example_driver['version'],
                                   example_driver_file=os.path.basename(example_driver_url))
 
+def windows_driver_rows(drivers):
+    driver_row_tmpl = template('windows_driver_row.tmpl', True)
+    markdown_link_tmpl = template('markdown_link.tmpl', True)
+    def row_gen():
+        for d in drivers:
+            product = PRODUCT_LABELS[Product[d['product']]]
+            variant = d.get('variant')
+            version_variant = d['version']
+            version_variant += (" " + variant) if variant else ''
+            patch64_url = d.get('patch64_url')
+            patch32_url = d.get('patch32_url')
+            driver_url = d.get('driver_url')
+            patch64_link = markdown_link_tmpl.substitute(text="Direct link",
+                                                         url=patch64_url) if patch64_url else ''
+            patch32_link = markdown_link_tmpl.substitute(text="Direct link",
+                                                         url=patch32_url) if patch32_url else ''
+            driver_link = markdown_link_tmpl.substitute(text="Direct link",
+                                                        url=driver_url) if driver_url else ''
+            yield driver_row_tmpl.substitute(product=product,
+                                             version_variant=version_variant,
+                                             patch64_link=patch64_link,
+                                             patch32_link=patch32_link,
+                                             driver_link=driver_link)
+    return "\n".join(row_gen())
+
+def windows_product_sections(drivers):
+    product_section_tmpl = template('windows_product_section.tmpl')
+    def section_gen():
+        for k, g in groupby(drivers, lambda d: Product[d['product']]):
+            driver_rows = windows_driver_rows(g)
+            yield product_section_tmpl.substitute(driver_rows=driver_rows)
+    return '\n\n'.join(section_gen())
+
+def windows_driver_table(drivers):
+    os_section_tmpl = template('windows_os_section.tmpl', True)
+    def section_gen():
+        for k, g in groupby(drivers, lambda d: WinSeries[d['os']]):
+            os = WIN_SERIES_LABELS[k]
+            product_sections = windows_product_sections(g)
+            yield os_section_tmpl.substitute(os=os,
+                                             product_sections=product_sections)
+    return '\n\n'.join(section_gen())
+
+def windows_readme(data):
+    master_tmpl = template('windows_readme_master.tmpl')
+    def driver_key_fun(d):
+        return ((WinSeries[d['os']], Product[d['product']]) +
+                version_key_fun(d['version']) +
+                (d.get('variant'),))
+    drivers = sorted(data['drivers'], key=driver_key_fun)
+    version_table = windows_driver_table(drivers)
+
+    geforce_drivers = filter(lambda d: Product[d['product']] is Product.GeForce, drivers)
+    quadro_drivers = filter(lambda d: Product[d['product']] is Product.Quadro, drivers)
+    latest_geforce_version = max(geforce_drivers, default='xxx.xx',
+                                 key=lambda d: version_key_fun(d['version']))['version']
+    latest_quadro_version = max(quadro_drivers, default='xxx.xx',
+                                key=lambda d: version_key_fun(d['version']))['version']
+    return master_tmpl.substitute(version_table=version_table,
+                                  latest_geforce_version=latest_geforce_version,
+                                  latest_quadro_version=latest_quadro_version)
+
 def main():
     with open(DATAFILE_PATH) as data_file:
         data = json.load(data_file)
     res = linux_readme(data['linux']['x86_64'])
     with open(LINUX_README_PATH, 'w', encoding=ENCODING) as out:
         out.write(res)
+    res = windows_readme(data['win']['x86_64'])
+    with open(WINDOWS_README_PATH, 'w', encoding=ENCODING) as out:
+        out.write(res)
 
 if __name__ == '__main__':
     main()

tools/readme_autogen/templates/markdown_link.tmpl

@@ -0,0 +1 @@
+[$text]($url)

tools/readme_autogen/templates/windows_driver_row.tmpl

@@ -0,0 +1 @@
+| $product | $version_variant | $patch64_link | $patch32_link | $driver_link |

tools/readme_autogen/templates/windows_os_section.tmpl

@@ -0,0 +1,4 @@
+### $os drivers
+
+
+$product_sections

tools/readme_autogen/templates/windows_product_section.tmpl

@@ -0,0 +1,3 @@
+| Product series | Version | x64 library patch | x86 library patch | Driver link |
+|----------------|---------|-------------------|-------------------|-------------|
+$driver_rows

tools/readme_autogen/templates/windows_readme_master.tmpl

@@ -0,0 +1,74 @@
+Nvidia drivers patch for Windows
+================================
+
+![GitHub last commit](https://img.shields.io/github/last-commit/keylase/nvidia-patch.svg) ![Latest GeForce version](https://img.shields.io/badge/latest%20GeForce%20version-${latest_geforce_version}-brightgreen.svg) ![Latest Quadro version](https://img.shields.io/badge/latest%20Quadro%20version-${latest_quadro_version}-blue.svg)
+
+This patch removes restriction on maximum number of simultaneous NVENC video encoding sessions imposed by Nvidia to consumer-grade GPUs.
+
+Requirements:
+
+- Any of following 64bit operating systems:
+  - Windows 7
+  - Windows 8
+  - Windows 8.1
+  - Windows 10
+  - Windows Server 2008 R2
+  - Windows Server 2012
+  - Windows Server 2012 R2
+  - Windows Server 2016
+  - Windows Server 2019
+- NVENC-compatible GPU (https://developer.nvidia.com/video-encode-decode-gpu-support-matrix#Encoder)
+- Nvidia driver. Patch available for versions in [table below](#version-table).
+
+## Step-by-Step Guide
+
+1. Download and install latest Nvidia driver supported by this patch.
+2. Download latest [Win\_1337\_Apply\_Patch tool](https://github.com/Deltafox79/Win_1337_Apply_Patch/releases/latest).
+3. Save appropriate patch(es) from [Version Table](#version-table) using direct link to the patch (Right Click -> Save as...). Alternatively you may checkout repo using git or download it as ZIP archive and then locate corresponding .1337 patch file in `win` directory.
+4. Apply x64 library patch to corresponding file in `%WINDIR%\system32\` with the Win\_1337\_Apply\_Patch tool. File name of patched object is specified in first line of .1337 patch. If x86 (32 bit) library patch is also available, apply it to same file in `%WINDIR%\SysWOW64\`.
+
+E.g, for 64bit Windows 10 running driver version $latest_geforce_version use `win10_x64/$latest_geforce_version/nvcuvid64.1337` against `C:\WINDOWS\system32\nvcuvid.dll` and `win10_x64/$latest_geforce_version/nvcuvid32.1337` against `C:\WINDOWS\SysWOW64\nvcuvid.dll`.
+
+~~There are additional steps may be required for Plex and 32bit apps users. See [corresponding section below](#d3d11-and-32-bit-apps-encoding-sessions).~~ We hope this is obsoleted by new additional x86 (32bit) library patch.
+
+A video tutorial is also available. Credits to designator2009. (*Covers pre-x86 patches. Now we probably don't need to autorun executable if x86 library patch applied*)
+
+[![Video Tutorial](https://gist.githubusercontent.com/Snawoot/de26b6ccfe67c7bc89ea4347d7c2ecde/raw/50cd87a72c4e13214e6c609dc5291037bed9db8d/ss.jpg)](https://www.youtube.com/watch?v=y7TRfRsJR-w)
+
+## Version Table
+
+$version_table
+
+## Benchmarks
+
+* [Plex Media Server: nVidia Hardware Transcoding Calculator for Plex Estimates](https://www.elpamsoft.com/?p=Plex-Hardware-Transcoding) - useful benchmark of achieved simultaneous transcodes with various stream quality and hardware with patched drivers.
+
+
+## See also
+
+* Genesis in [related issue](https://github.com/keylase/nvidia-patch/issues/9)
+
+### D3D11 and 32-bit apps encoding sessions
+
+This section is actual only for D3D11 encoders and earlier driver versions (before 440.97).
+
+This patch for earlier driver versions (those which do not have additional 32bit library patch) wasn't covering 32bit driver libraries and for this reason 32bit applications were limited unless limit is not raised by some 64bit applications. But once usage limit was exceeded, it persists for all kinds of apps until system reboot. So, for example, you may once open 10 sessions with 64bit version of `ffmpeg` and limit will get raised to 10 for all rest types of apps until reboot. You may follow these steps to achieve this automatically and have all limits raised (assuming patch above already applied):
+
+#### Method 1 (recommended)
+
+1. Download and run [latest release](https://github.com/jantenhove/NvencSessionLimitBump/releases) of [NvencSessionLimitBump](https://github.com/jantenhove/NvencSessionLimitBump).
+2. (Optional) Add it to autostart programs.
+
+By default this application raises limit to 32 encoding sessions. Credits to @jantenhove.
+
+#### Method 2 (alternative)
+
+1. Download 64bit FFmpeg for Windows: https://ffmpeg.zeranoe.com/builds/
+2. Unpack it somewhere.
+3. Get [`ffmpeg_null_10streams.cmd`](ffmpeg_null_10streams.cmd) from this repo.
+4. Edit `ffmpeg_null_10streams.cmd` and set executable path to real location of your unpacked ffmpeg.
+5. (Optional) Add `ffmpeg_null_10streams.cmd` to autostart programs.
+
+**Bonus**: you may use [this AutoIt script](silent_bump.au3) from @wazerstar for silent startup of console applications bumping sessions.
+
+Also you may use these methods to check if patch applied correctly and limit was raised. Use them when nothing works and you are in doubt.