mirror of https://github.com/opnsense/docs
add collect_changelogs.py script to collect and transform release logs.
Ad Schellevis
4 years ago
parent
5372a5bcad
commit
74a66de4c6
@ -0,0 +1,155 @@
|
||||
#!/usr/local/bin/python
|
||||
"""
|
||||
Copyright (c) 2020 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import requests
|
||||
import zipfile
|
||||
import collections
|
||||
import re
|
||||
from packaging import version
|
||||
from jinja2 import Template
|
||||
|
||||
def download_zip(target_filename):
|
||||
req = requests.get("https://github.com/opnsense/changelog/archive/master.zip", stream=True)
|
||||
if req.status_code == 200:
|
||||
req.raw.decode_content = True
|
||||
with open(target_filename, 'wb') as f_out:
|
||||
while True:
|
||||
data = req.raw.read(10240)
|
||||
if not data:
|
||||
break
|
||||
else:
|
||||
f_out.write(data)
|
||||
|
||||
def parse_change_log(payload, this_version):
|
||||
result = {
|
||||
"release_date": "---",
|
||||
"prelude": list(),
|
||||
"content": list()
|
||||
}
|
||||
all_tokens = set()
|
||||
all_token_links = dict()
|
||||
first_line = False
|
||||
prelude_line = this_version.count(".") == 1
|
||||
rst_content = list()
|
||||
lines = payload.split("\n")
|
||||
for idx, line in enumerate(lines):
|
||||
content_line = None
|
||||
# general cleanups
|
||||
line = line.replace('*', '\*')
|
||||
if line.find('`') > -1:
|
||||
line = re.sub(r'(`)([^`|\']*)([`|\'])', r':code:`\2`', line)
|
||||
#
|
||||
for token in re.findall(r'(\[[0-9]{1,2}\])', line):
|
||||
all_tokens.add(token)
|
||||
if idx < 3 and line.find('@') > -1:
|
||||
result["release_date"] = line.split('@')[1].strip()
|
||||
elif first_line is False and line.strip() != "":
|
||||
# strip tag line
|
||||
first_line = idx
|
||||
if line.find('OPNsense') > -1:
|
||||
content_line = line
|
||||
elif line.startswith('o '):
|
||||
content_line = "*%s" % line[1:]
|
||||
elif line.startswith('# '):
|
||||
if not lines[idx-1].startswith('# '):
|
||||
content_line = ".. code-block::\n\n %s" % line
|
||||
else:
|
||||
content_line = " %s" % line
|
||||
elif line.startswith('[') and line[0:line.find(']')+1] in all_tokens:
|
||||
token = line[0:line.find(']')+1]
|
||||
all_token_links[token] = line[len(token)+1:].strip()
|
||||
else:
|
||||
content_line = line
|
||||
|
||||
if content_line is not None:
|
||||
result['content'].append(content_line)
|
||||
if prelude_line:
|
||||
result['prelude'].append(content_line)
|
||||
|
||||
# prelude exit
|
||||
if prelude_line and line.find('https://opnsense.org/download/') > -1:
|
||||
prelude_line = False
|
||||
|
||||
result["content"] = "\n".join(result["content"])
|
||||
result["prelude"] = "\n".join(result["prelude"])
|
||||
# replace links
|
||||
for section in ['content', 'prelude']:
|
||||
for token in all_token_links:
|
||||
result[section] = result[section].replace(token, " `%s <%s>`__ " % (token, all_token_links[token]))
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--skip_download', help='skip downloading github rep', action='store_true')
|
||||
cmd_args = parser.parse_args()
|
||||
|
||||
root_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
changelog_zip = "%s/changelog_github.zip" % root_dir
|
||||
if not cmd_args.skip_download:
|
||||
download_zip(changelog_zip)
|
||||
|
||||
if os.path.isfile(changelog_zip):
|
||||
template_data = {
|
||||
'major_versions': collections.OrderedDict(),
|
||||
'versions': collections.OrderedDict(),
|
||||
'nicknames': collections.OrderedDict(),
|
||||
}
|
||||
all_versions = dict()
|
||||
# read all changelogs (from zip)
|
||||
with zipfile.ZipFile(changelog_zip, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for item in zf.infolist():
|
||||
fparts = item.filename.split('/')
|
||||
if len(fparts) > 3 and fparts[1] == 'doc' and item.file_size > 0:
|
||||
all_versions[fparts[3]] = zf.open(item).read().decode()
|
||||
|
||||
for my_version in sorted(all_versions, key=lambda x: version.Version(x.replace('.r','rc')), reverse=True):
|
||||
major_version = ".".join(my_version.split('.')[:2])
|
||||
if major_version not in template_data['major_versions']:
|
||||
template_data['major_versions'][major_version] = dict()
|
||||
template_data['versions'][my_version] = parse_change_log(all_versions[my_version], my_version)
|
||||
|
||||
if major_version == my_version:
|
||||
template_data['nicknames'][my_version] = ""
|
||||
tmp = all_versions[my_version].replace('\n', ' ')
|
||||
m = re.match(r'.*nicknamed ["\'](?P<nickname>[^"\']+)', tmp)
|
||||
if m:
|
||||
template_data['nicknames'][my_version] = m.groupdict()['nickname']
|
||||
|
||||
# root menu index
|
||||
with open("%s/source/releases.rst" % root_dir, 'w') as f_out:
|
||||
template = Template(open("%s/source/releases.rst.in" % root_dir, "r").read())
|
||||
f_out.write(template.render(template_data))
|
||||
|
||||
# per version rst file
|
||||
template = Template(open("%s/source/releases/default.rst.in" % root_dir, "r").read())
|
||||
for major_version in template_data['major_versions']:
|
||||
with open("%s/source/releases/%s.rst" % (root_dir, major_version), 'w') as f_out:
|
||||
template_data['this_version'] = major_version
|
||||
f_out.write(template.render(template_data))
|
||||
@ -0,0 +1,20 @@
|
||||
====================================
|
||||
Releases
|
||||
====================================
|
||||
|
||||
.. image:: /development/images/ideas_join_the_development.jpg
|
||||
:width: 600px
|
||||
:align: center
|
||||
|
||||
As of January 2015 there have been *{{ versions|length }}* releases leading to the latest version *{{ versions|first }}*
|
||||
named "{{nicknames.items()|list|first|last}}".
|
||||
|
||||
The list below contains all releases, ordered by version number categorized by major version.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:titlesonly:
|
||||
:glob:
|
||||
{% for major_version in major_versions%}
|
||||
releases/{{major_version}}
|
||||
{%- endfor %}
|
||||
@ -0,0 +1,14 @@
|
||||
===========================================================================================
|
||||
{{ this_version }} "{{ nicknames[this_version] }}" Series
|
||||
===========================================================================================
|
||||
|
||||
{{ versions[this_version]['prelude'] }}
|
||||
|
||||
{% for version in versions %}
|
||||
{%- if version.find(this_version) == 0 %}
|
||||
--------------------------------------------------------------------------
|
||||
{{ version }} ({{versions[version]['release_date']}})
|
||||
--------------------------------------------------------------------------
|
||||
{{ versions[version]['content'] }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
Loading…
Reference in New Issue