From 1207e650af4a0fd839dd4c5e7a160180e999373f Mon Sep 17 00:00:00 2001 From: Vladislav Yarmak Date: Sat, 31 Oct 2020 18:06:50 +0200 Subject: [PATCH] ndl: implement deb repo channel --- tools/nv-driver-locator/README.md | 241 +----------------- .../nv-driver-locator.json.sample | 63 ++++- tools/nv-driver-locator/nv-driver-locator.py | 87 +++++-- 3 files changed, 135 insertions(+), 256 deletions(-) diff --git a/tools/nv-driver-locator/README.md b/tools/nv-driver-locator/README.md index b250fdf..a795694 100644 --- a/tools/nv-driver-locator/README.md +++ b/tools/nv-driver-locator/README.md @@ -35,232 +35,7 @@ All scripts may be used both as standalone application and importable module. Fo ## Configuration example -```json -{ - "db": { - "type": "file", - "params": { - "workdir": "/var/lib/nv-driver-locator" - } - }, - "key_components": [ - [ - "DriverAttributes", - "Version" - ], - [ - "DriverAttributes", - "Name" - ], - [ - "ChannelAttributes", - "OS" - ] - ], - "channels": [ - { - "type": "gfe_client", - "name": "desktop defaults", - "params": {} - }, - { - "type": "gfe_client", - "name": "desktop beta", - "params": { - "beta": true - } - }, - { - "type": "gfe_client", - "name": "mobile", - "params": { - "notebook": true - } - }, - { - "type": "gfe_client", - "name": "mobile beta", - "params": { - "notebook": true, - "beta": true - } - }, - { - "type": "nvidia_downloads", - "name": "linux beta", - "params": { - "os": "Linux_64", - "product": "GeForce", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux stable", - "params": { - "os": "Linux_64", - "product": "GeForce", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win stable", - "params": { - "os": "Windows10_64", - "product": "GeForce", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win beta", - "params": { - "os": "Windows10_64", - "product": "GeForce", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win notebook stable", - "params": { - "os": "Windows10_64", - "product": "GeForceMobile", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win notebook beta", - "params": { - "os": "Windows10_64", - "product": "GeForceMobile", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux quadro beta", - "params": { - "os": "Linux_64", - "product": "Quadro", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux quadro stable", - "params": { - "os": "Linux_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro stable", - "params": { - "os": "Windows10_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro beta", - "params": { - "os": "Windows10_64", - "product": "Quadro", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro notebook stable", - "params": { - "os": "Windows10_64", - "product": "QuadroMobile", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro notebook beta", - "params": { - "os": "Windows10_64", - "product": "QuadroMobile", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win server quadro certified", - "params": { - "os": "WindowsServer2012R2_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win server 2019 quadro certified", - "params": { - "os": "WindowsServer2019", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "cuda_downloads", - "name": "cuda toolkit tracker", - "params": {} - }, - { - "type": "vulkan_beta", - "name": "vulkan beta windows", - "params": { - "os": "Windows" - } - }, - { - "type": "vulkan_beta", - "name": "vulkan beta linux", - "params": { - "os": "Linux" - } - } - ], - "notifiers": [ - { - "type": "email", - "name": "my email", - "params": { - "from_addr": "notify-bot@gmail.com", - "to_addrs": [ - "recepient1@domain1.tld", - "recepient2@domain2.tld" - ], - "host": "smtp.google.com", - "use_starttls": true, - "login": "notify-bot", - "password": "MyGoodPass" - } - }, - { - "type": "command", - "name": "sample command", - "params": { - "timeout": 10.0, - "cmdline": [ - "cat", - "-" - ] - } - } - ] -} -``` +See [nv-driver-locator.json.sample](nv-driver-locator.json.sample). ## Components Reference @@ -329,7 +104,19 @@ Type: `vulkan_beta` Params: -* `os` - OS family. Allowed values: `Linux`, `Windows`. Default: `Linux`. +* `timeout` - allowed delay in seconds for each network operation. Default: `10.0` + +#### DebRepoChannel + +Parses Packages or Packages.gz file of deb package repository and extracts versions information. + +Type: `deb_packages` + +Params: + +* `url` - Location of `Packages` or `Packages.gz` file. +* `pkg_pattern` - regexp which defines package name pattern. +* `driver_name` - value returned as `DriverAttributes.Name` for proper aggregation by Hasher. Default: `"Linux x64 (AMD64/EM64T) Display Driver"` * `timeout` - allowed delay in seconds for each network operation. Default: `10.0` ### Notifiers diff --git a/tools/nv-driver-locator/nv-driver-locator.json.sample b/tools/nv-driver-locator/nv-driver-locator.json.sample index 9d1061c..ed5514d 100644 --- a/tools/nv-driver-locator/nv-driver-locator.json.sample +++ b/tools/nv-driver-locator/nv-driver-locator.json.sample @@ -242,15 +242,70 @@ { "type": "vulkan_beta", "name": "vulkan beta windows", + "params": {} + }, + { + "type": "deb_packages", + "name": "nvidia cuda drivers deb repo for ubuntu 18.04", "params": { - "os": "Windows" + "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/Packages.gz", + "pkg_pattern": "^cuda-drivers$" } }, { - "type": "vulkan_beta", - "name": "vulkan beta linux", + "type": "deb_packages", + "name": "nvidia cuda drivers deb repo for ubuntu 20.04", + "params": { + "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/Packages.gz", + "pkg_pattern": "^cuda-drivers$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos (updates)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic-updates/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos (security)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic-security/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/focal/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos (updates)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/focal-updates/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos (security)", "params": { - "os": "Linux" + "url": "http://security.ubuntu.com/ubuntu/dists/focal-security/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" } } ], diff --git a/tools/nv-driver-locator/nv-driver-locator.py b/tools/nv-driver-locator/nv-driver-locator.py index d94cfad..ab67eb1 100755 --- a/tools/nv-driver-locator/nv-driver-locator.py +++ b/tools/nv-driver-locator/nv-driver-locator.py @@ -179,14 +179,14 @@ class GFEClientChannel(BaseChannel): def get_latest_drivers(self): res = self._get_latest_drivers(notebook=self._notebook, - x86_64=self._x86_64, - os_version=self._os_version, - os_build=self._os_build, - language=self._language, - beta=self._beta, - dch=self._dch, - crd=self._crd, - timeout=self._timeout) + x86_64=self._x86_64, + os_version=self._os_version, + os_build=self._os_build, + language=self._language, + beta=self._beta, + dch=self._dch, + crd=self._crd, + timeout=self._timeout) if res is None: return res.update({ @@ -275,36 +275,72 @@ class CudaToolkitDownloadsChannel(BaseChannel): 'Version': '???', 'Name': latest, 'NameLocalized': latest, + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, } } -@functools.lru_cache(maxsize=0) -def vulkan_downloads(*, timeout=10): - gvd = importlib.import_module('get_vulkan_downloads') - return gvd.get_drivers(timeout=timeout) - class VulkanBetaDownloadsChannel(BaseChannel): def __init__(self, name, *, - os="Linux", timeout=10): self.name = name - self._os = os self._timeout = timeout + self._gvd = importlib.import_module('get_vulkan_downloads') def get_latest_drivers(self): - drivers = vulkan_downloads(timeout=self._timeout) + drivers = self._gvd.get_drivers(timeout=self._timeout) + if drivers is None: + return for drv in drivers: - if drv["os"] == self._os: - yield { - 'DriverAttributes': { - 'Version': drv['version'], - 'Name': drv['name'], - 'NameLocalized': drv['name'], - } + yield { + 'DriverAttributes': { + 'Version': drv['version'], + 'Name': drv['name'], + 'NameLocalized': drv['name'], + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, + 'OS': drv['os'], } - else: - return + } +class DebRepoChannel(BaseChannel): + def __init__(self, name, *, + url, + pkg_pattern, + driver_name="Linux x64 (AMD64/EM64T) Display Driver", + timeout=10): + self.name = name + self._gdd = importlib.import_module('get_deb_drivers') + self._url = url + self._pkg_pattern = pkg_pattern + self._driver_name = driver_name + self._timeout = timeout + + def get_latest_drivers(self): + drivers = self._gdd.get_deb_versions(url=self._url, + name=self._pkg_pattern, + timeout=self._timeout) + if drivers is None: + return + for drv in drivers: + yield { + 'DriverAttributes': { + 'Version': drv.version, + 'DebPkgName': drv.name, + 'Name': self._driver_name, + 'NameLocalized': self._driver_name, + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, + 'OS': 'Linux_64', + 'PkgPattern': self._pkg_pattern, + } + } def parse_args(): parser = argparse.ArgumentParser( @@ -334,6 +370,7 @@ class DriverLocator: 'nvidia_downloads': NvidiaDownloadsChannel, 'cuda_downloads': CudaToolkitDownloadsChannel, 'vulkan_beta': VulkanBetaDownloadsChannel, + 'deb_packages': DebRepoChannel, } channels = []