meli/melib/build.rs

/*
 * meli - melib crate.
 *
 * Copyright 2017-2020 Manos Pitsidianakis
 *
 * This file is part of meli.
 *
 * meli is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * meli is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with meli. If not, see <http://www.gnu.org/licenses/>.
 */

#[cfg(feature = "unicode_algorithms")]
include!("src/text_processing/types.rs");

fn main() -> Result<(), std::io::Error> {
    #[cfg(feature = "unicode_algorithms")]
    {
        const MOD_PATH: &str = "src/text_processing/tables.rs";
        println!("cargo:rerun-if-changed=build.rs");
        println!("cargo:rerun-if-changed={}", MOD_PATH);
        /* Line break tables */
        use std::fs::File;
        use std::io::prelude::*;
        use std::io::BufReader;
        use std::path::Path;
        use std::process::{Command, Stdio};
        const LINE_BREAK_TABLE_URL: &str =
            "http://www.unicode.org/Public/UCD/latest/ucd/LineBreak.txt";
        /* Grapheme width tables */
        const UNICODE_DATA_URL: &str =
            "http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt";
        const EAW_URL: &str = "http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt";
        const EMOJI_DATA_URL: &str =
            "https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt";

        let mod_path = Path::new(MOD_PATH);
        if mod_path.exists() {
            eprintln!(
                "{} already exists, delete it if you want to replace it.",
                mod_path.display()
            );
            std::process::exit(0);
        }
        let mut child = Command::new("curl")
            .args(&["-o", "-", LINE_BREAK_TABLE_URL])
            .stdout(Stdio::piped())
            .stdin(Stdio::null())
            .stderr(Stdio::inherit())
            .spawn()?;

        let buf_reader = BufReader::new(child.stdout.take().unwrap());

        let mut line_break_table: Vec<(u32, u32, LineBreakClass)> = Vec::with_capacity(3800);
        for line in buf_reader.lines() {
            let line = line.unwrap();
            if line.starts_with('#') || line.starts_with(' ') || line.is_empty() {
                continue;
            }
            let tokens: &str = line.split_whitespace().next().unwrap();

            let semicolon_idx: usize = tokens.chars().position(|c| c == ';').unwrap();
            /* LineBreak.txt list is ascii encoded so we can assume each char takes one byte: */
            let chars_str: &str = &tokens[..semicolon_idx];

            let mut codepoint_iter = chars_str.split("..");

            let first_codepoint: u32 =
                u32::from_str_radix(codepoint_iter.next().unwrap(), 16).unwrap();

            let sec_codepoint: u32 = codepoint_iter
                .next()
                .map(|v| u32::from_str_radix(v, 16).unwrap())
                .unwrap_or(first_codepoint);
            let class = &tokens[semicolon_idx + 1..semicolon_idx + 1 + 2];
            line_break_table.push((first_codepoint, sec_codepoint, LineBreakClass::from(class)));
        }
        child.wait()?;

        let child = Command::new("curl")
            .args(&["-o", "-", UNICODE_DATA_URL])
            .stdout(Stdio::piped())
            .output()?;

        let unicode_data = String::from_utf8_lossy(&child.stdout);

        let child = Command::new("curl")
            .args(&["-o", "-", EAW_URL])
            .stdout(Stdio::piped())
            .output()?;

        let eaw_data = String::from_utf8_lossy(&child.stdout);

        let child = Command::new("curl")
            .args(&["-o", "-", EMOJI_DATA_URL])
            .stdout(Stdio::piped())
            .output()?;

        let emoji_data = String::from_utf8_lossy(&child.stdout);

        const MAX_CODEPOINT: usize = 0x110000;
        // See https://www.unicode.org/L2/L1999/UnicodeData.html
        const FIELD_CODEPOINT: usize = 0;
        const FIELD_CATEGORY: usize = 2;
        // Ambiguous East Asian characters
        const WIDTH_AMBIGUOUS_EASTASIAN: isize = -3;

        // Width changed from 1 to 2 in Unicode 9.0
        const WIDTH_WIDENED_IN_9: isize = -6;
        // Category for unassigned codepoints.
        const CAT_UNASSIGNED: &str = "Cn";

        // Category for private use codepoints.
        const CAT_PRIVATE_USE: &str = "Co";

        // Category for surrogates.
        const CAT_SURROGATE: &str = "Cs";

        struct Codepoint<'cat> {
            raw: u32,
            width: Option<isize>,
            category: &'cat str,
        }

        let mut codepoints: Vec<Codepoint> = Vec::with_capacity(MAX_CODEPOINT + 1);
        for i in 0..=MAX_CODEPOINT {
            codepoints.push(Codepoint {
                raw: i as u32,
                width: None,
                category: CAT_UNASSIGNED,
            });
        }

        set_general_categories(&mut codepoints, &unicode_data);
        set_eaw_widths(&mut codepoints, &eaw_data);
        set_emoji_widths(&mut codepoints, &emoji_data);
        set_hardcoded_ranges(&mut codepoints);
        fn hexrange_to_range(hexrange: &str) -> std::ops::Range<usize> {
            /* Given a string like 1F300..1F320 representing an inclusive range,
            return the range of codepoints.
            If the string is like 1F321, return a range of just that element.
            */
            let hexrange = hexrange.trim();
            let fields = hexrange
                .split("..")
                .map(|h| usize::from_str_radix(h.trim(), 16).unwrap())
                .collect::<Vec<usize>>();
            if fields.len() == 1 {
                fields[0]..(fields[0] + 1)
            } else {
                fields[0]..(fields[1] + 1)
            }
        }

        fn set_general_categories<'u>(codepoints: &mut [Codepoint<'u>], unicode_data: &'u str) {
            for line in unicode_data.lines() {
                let fields = line.trim().split(';').collect::<Vec<_>>();
                if fields.len() > FIELD_CATEGORY {
                    for idx in hexrange_to_range(fields[FIELD_CODEPOINT]) {
                        codepoints[idx].category = fields[FIELD_CATEGORY];
                    }
                }
            }
        }

        fn set_eaw_widths(codepoints: &mut [Codepoint<'_>], eaw_data_lines: &str) {
            //  Read from EastAsianWidth.txt, set width values on the codepoints
            for line in eaw_data_lines.lines() {
                let line = line.trim().split('#').next().unwrap_or(line);
                let fields = line.trim().split(';').collect::<Vec<_>>();
                if fields.len() != 2 {
                    continue;
                }
                let hexrange = fields[0];
                let width_type = fields[1];
                // width_types:
                //  A: ambiguous, F: fullwidth, H: halfwidth,
                // . N: neutral, Na: east-asian Narrow
                let width: isize = if width_type == "A" {
                    WIDTH_AMBIGUOUS_EASTASIAN
                } else if width_type == "F" || width_type == "W" {
                    2
                } else {
                    1
                };
                for cp in hexrange_to_range(hexrange) {
                    codepoints[cp].width = Some(width);
                }
            }
            // Apply the following special cases:
            //  - The unassigned code points in the following blocks default to "W":
            //         CJK Unified Ideographs Extension A: U+3400..U+4DBF
            //         CJK Unified Ideographs:             U+4E00..U+9FFF
            //         CJK Compatibility Ideographs:       U+F900..U+FAFF
            //  - All undesignated code points in Planes 2 and 3, whether inside or
            //      outside of allocated blocks, default to "W":
            //         Plane 2:                            U+20000..U+2FFFD
            //         Plane 3:                            U+30000..U+3FFFD
            const WIDE_RANGES: [(usize, usize); 5] = [
                (0x3400, 0x4DBF),
                (0x4E00, 0x9FFF),
                (0xF900, 0xFAFF),
                (0x20000, 0x2FFFD),
                (0x30000, 0x3FFFD),
            ];
            for &wr in WIDE_RANGES.iter() {
                for cp in wr.0..(wr.1 + 1) {
                    if codepoints[cp].width.is_none() {
                        codepoints[cp].width = Some(2);
                    }
                }
            }
        }

        fn set_emoji_widths(codepoints: &mut [Codepoint<'_>], emoji_data_lines: &str) {
            // Read from emoji-data.txt, set codepoint widths
            for line in emoji_data_lines.lines() {
                if !line.contains('#') || line.trim().starts_with('#') {
                    continue;
                }
                let mut fields = line.trim().split('#').collect::<Vec<_>>();
                if fields.len() != 2 {
                    continue;
                }
                let comment = fields.pop().unwrap();
                let fields = fields.pop().unwrap();

                let hexrange = fields.split(';').next().unwrap();

                // In later versions of emoji-data.txt there are some "reserved"
                // entries that have "NA" instead of a Unicode version number
                // of first use, they will now return a zero version instead of
                // crashing the script
                if comment.trim().starts_with("NA") {
                    continue;
                }

                use std::str::FromStr;
                let mut v = comment.trim().split_whitespace().next().unwrap();
                if v.starts_with('E') {
                    v = &v[1..];
                }
                if v.as_bytes()
                    .get(0)
                    .map(|c| !c.is_ascii_digit())
                    .unwrap_or(true)
                {
                    continue;
                }
                let mut idx = 1;
                while v
                    .as_bytes()
                    .get(idx)
                    .map(|c| c.is_ascii_digit())
                    .unwrap_or(false)
                {
                    idx += 1;
                }
                if v.as_bytes().get(idx).map(|&c| c != b'.').unwrap_or(true) {
                    continue;
                }
                idx += 1;
                while v
                    .as_bytes()
                    .get(idx)
                    .map(|c| c.is_ascii_digit())
                    .unwrap_or(false)
                {
                    idx += 1;
                }
                v = &v[0..idx];

                let version = f32::from_str(v).unwrap();
                for cp in hexrange_to_range(hexrange) {
                    // Don't consider <=1F000 values as emoji. These can only be made
                    // emoji through the variation selector which interacts terribly
                    // with wcwidth().
                    if cp < 0x1F000 {
                        continue;
                    }
                    // Skip codepoints that are explicitly not wide.
                    // For example U+1F336 ("Hot Pepper") renders like any emoji but is
                    // marked as neutral in EAW so has width 1 for some reason.
                    //if codepoints[cp].width == Some(1) {
                    //    continue;
                    //}

                    // If this emoji was introduced before Unicode 9, then it was widened in 9.
                    codepoints[cp].width = if version >= 9.0 {
                        Some(2)
                    } else {
                        Some(WIDTH_WIDENED_IN_9)
                    };
                }
            }
        }
        fn set_hardcoded_ranges(codepoints: &mut [Codepoint<'_>]) {
            // Mark private use and surrogate codepoints
            // Private use can be determined awkwardly from UnicodeData.txt,
            // but we just hard-code them.
            // We do not treat "private use high surrogate" as private use
            // so as to match wcwidth9().
            const PRIVATE_RANGES: [(usize, usize); 3] =
                [(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)];
            for &(first, last) in PRIVATE_RANGES.iter() {
                for idx in first..=last {
                    codepoints[idx].category = CAT_PRIVATE_USE;
                }
            }

            const SURROGATE_RANGES: [(usize, usize); 2] = [(0xD800, 0xDBFF), (0xDC00, 0xDFFF)];
            for &(first, last) in SURROGATE_RANGES.iter() {
                for idx in first..=last {
                    codepoints[idx].category = CAT_SURROGATE;
                }
            }
        }

        let mut file = File::create(&mod_path)?;
        file.write_all(
            br#"/*
 * meli - text_processing crate.
 *
 * Copyright 2017-2020 Manos Pitsidianakis
 *
 * This file is part of meli.
 *
 * meli is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * meli is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with meli. If not, see <http://www.gnu.org/licenses/>.
 */

use super::types::LineBreakClass::{self, *};

pub const LINE_BREAK_RULES: &[(u32, u32, LineBreakClass)] = &[
"#,
        )
        .unwrap();
        for l in &line_break_table {
            file.write_all(format!("    (0x{:X}, 0x{:X}, {:?}),\n", l.0, l.1, l.2).as_bytes())
                .unwrap();
        }
        file.write_all(b"];\n").unwrap();

        for (name, filter) in [
            (
                "ASCII",
                Box::new(|c: &&Codepoint| c.raw < 0x7f && c.raw >= 0x20)
                    as Box<dyn Fn(&&Codepoint) -> bool>,
            ),
            (
                "PRIVATE",
                Box::new(|c: &&Codepoint| c.category == CAT_PRIVATE_USE),
            ),
            (
                "NONPRINT",
                Box::new(|c: &&Codepoint| {
                    ["Cc", "Cf", "Zl", "Zp", CAT_SURROGATE].contains(&c.category)
                }),
            ),
            (
                "COMBINING",
                Box::new(|c: &&Codepoint| ["Mn", "Mc", "Me"].contains(&c.category)),
            ),
            ("DOUBLEWIDE", Box::new(|c: &&Codepoint| c.width == Some(2))),
            (
                "UNASSIGNED",
                Box::new(|c: &&Codepoint| c.category == CAT_UNASSIGNED),
            ),
            (
                "AMBIGUOUS",
                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_AMBIGUOUS_EASTASIAN)),
            ),
            (
                "WIDENEDIN9",
                Box::new(|c: &&Codepoint| c.width == Some(WIDTH_WIDENED_IN_9)),
            ),
        ]
        .iter()
        {
            file.write_all(
                format!(
                    r#"
pub const {}: &[(u32, u32)] = &[
"#,
                    name
                )
                .as_bytes(),
            )
            .unwrap();
            let mut iter = codepoints.iter().filter(filter);
            let mut prev = iter.next().unwrap().raw;
            let mut a = prev;
            for cp in iter {
                if prev + 1 != cp.raw {
                    file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
                        .unwrap();
                    a = cp.raw;
                }
                prev = cp.raw;
            }
            file.write_all(format!("    (0x{:X}, 0x{:X}),\n", a, prev).as_bytes())
                .unwrap();
            file.write_all(b"];\n").unwrap();
        }
    }
    Ok(())
}