mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-10-30 21:20:34 +00:00
[outtmpl] Format type U
for unicode normalization
This commit is contained in:
parent
f440b14f87
commit
524e2e4fda
@ -964,6 +964,7 @@ # OUTPUT TEMPLATE
|
||||
1. **Alternatives**: Alternate fields can be specified seperated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
|
||||
1. **Default**: A literal default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
|
||||
1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son, a comma seperated **l**ist and a string **q**uoted for the terminal respectively
|
||||
1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
|
||||
|
||||
To summarize, the general syntax for a field is:
|
||||
```
|
||||
|
@ -649,7 +649,7 @@ def test_add_extra_info(self):
|
||||
'title2': '%PATH%',
|
||||
'title3': 'foo/bar\\test',
|
||||
'title4': 'foo "bar" test',
|
||||
'title5': 'áéí',
|
||||
'title5': 'áéí 𝐀',
|
||||
'timestamp': 1618488000,
|
||||
'duration': 100000,
|
||||
'playlist_index': 1,
|
||||
@ -769,6 +769,10 @@ def expect_same_infodict(out):
|
||||
test('%(formats.:.id) 15l', ' id1, id2, id3')
|
||||
test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
|
||||
test('%(title5).3B', 'á')
|
||||
test('%(title5)U', 'áéí 𝐀')
|
||||
test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
|
||||
test('%(title5)+U', 'áéí A')
|
||||
test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
|
||||
if compat_os_name == 'nt':
|
||||
test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
|
||||
else:
|
||||
|
@ -25,6 +25,7 @@
|
||||
import tokenize
|
||||
import traceback
|
||||
import random
|
||||
import unicodedata
|
||||
|
||||
from string import ascii_letters
|
||||
|
||||
@ -908,7 +909,7 @@ def escape_outtmpl(outtmpl):
|
||||
def validate_outtmpl(cls, outtmpl):
|
||||
''' @return None or Exception object '''
|
||||
outtmpl = re.sub(
|
||||
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljqB]'),
|
||||
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljqBU]'),
|
||||
lambda mobj: f'{mobj.group(0)[:-1]}s',
|
||||
cls._outtmpl_expandpath(outtmpl))
|
||||
try:
|
||||
@ -940,7 +941,7 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
|
||||
}
|
||||
|
||||
TMPL_DICT = {}
|
||||
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljqB]'))
|
||||
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljqBU]'))
|
||||
MATH_FUNCTIONS = {
|
||||
'+': float.__add__,
|
||||
'-': float.__sub__,
|
||||
@ -1031,21 +1032,26 @@ def create_key(outer_mobj):
|
||||
value = default if value is None else value
|
||||
|
||||
str_fmt = f'{fmt[:-1]}s'
|
||||
if fmt[-1] == 'l':
|
||||
if fmt[-1] == 'l': # list
|
||||
value, fmt = ', '.join(variadic(value)), str_fmt
|
||||
elif fmt[-1] == 'j':
|
||||
elif fmt[-1] == 'j': # json
|
||||
value, fmt = json.dumps(value, default=_dumpjson_default), str_fmt
|
||||
elif fmt[-1] == 'q':
|
||||
elif fmt[-1] == 'q': # quoted
|
||||
value, fmt = compat_shlex_quote(str(value)), str_fmt
|
||||
elif fmt[-1] == 'B':
|
||||
elif fmt[-1] == 'B': # bytes
|
||||
value = f'%{str_fmt}'.encode('utf-8') % str(value).encode('utf-8')
|
||||
value, fmt = value.decode('utf-8', 'ignore'), 's'
|
||||
elif fmt[-1] == 'U': # unicode normalized
|
||||
opts = outer_mobj.group('conversion') or ''
|
||||
value, fmt = unicodedata.normalize(
|
||||
# "+" = compatibility equivalence, "#" = NFD
|
||||
'NF%s%s' % ('K' if '+' in opts else '', 'D' if '#' in opts else 'C'),
|
||||
value), str_fmt
|
||||
elif fmt[-1] == 'c':
|
||||
value = str(value)
|
||||
if value is None:
|
||||
value, fmt = default, 's'
|
||||
if value:
|
||||
value = str(value)[0]
|
||||
else:
|
||||
value = value[0]
|
||||
fmt = str_fmt
|
||||
elif fmt[-1] not in 'rs': # numeric
|
||||
value = float_or_none(value)
|
||||
if value is None:
|
||||
|
@ -4474,12 +4474,12 @@ def q(qid):
|
||||
STR_FORMAT_RE_TMPL = r'''(?x)
|
||||
(?<!%)(?P<prefix>(?:%%)*)
|
||||
%
|
||||
(?P<has_key>\((?P<key>{0})\))? # mapping key
|
||||
(?P<has_key>\((?P<key>{0})\))?
|
||||
(?P<format>
|
||||
(?:[#0\-+ ]+)? # conversion flags (optional)
|
||||
(?:\d+)? # minimum field width (optional)
|
||||
(?:\.\d+)? # precision (optional)
|
||||
[hlL]? # length modifier (optional)
|
||||
(?P<conversion>[#0\-+ ]+)?
|
||||
(?P<min_width>\d+)?
|
||||
(?P<precision>\.\d+)?
|
||||
(?P<len_mod>[hlL])? # unused in python
|
||||
{1} # conversion type
|
||||
)
|
||||
'''
|
||||
|
Loading…
Reference in New Issue
Block a user