Compare commits
No commits in common. "52ea2745dd9e7b12400de3ceb6d6901962514b03" and "b740e95a62397e5b8aedd459ab65d814dc43cd36" have entirely different histories.
52ea2745dd
...
b740e95a62
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
NlpSrc/*.nlp binary
|
||||
389
.gitignore
vendored
389
.gitignore
vendored
@ -1,16 +1,373 @@
|
||||
## Global ignore
|
||||
# out/
|
||||
# temp/
|
||||
# NlpSrc/*
|
||||
# !NlpSrc/*.nlp
|
||||
# !NlpSrc/README.md
|
||||
# NlpParser/*
|
||||
# !NlpParser/Nlp.g4
|
||||
# !NlpParser/NlpRunner.java
|
||||
# !NlpParser/testbench.txt
|
||||
# !NlpParser/README.md
|
||||
# NlpTr/*
|
||||
# !NlpTr/*.diff
|
||||
# !NlpTr/*.index
|
||||
# !NlpTr/*.json
|
||||
# !NlpTr/README.md
|
||||
## my ban
|
||||
out/
|
||||
temp/
|
||||
NlpSrc/*
|
||||
!NlpSrc/*.nlp
|
||||
!NlpSrc/README.md
|
||||
NlpParser/*
|
||||
!NlpParser/Nlp.g4
|
||||
!NlpParser/NlpRunner.java
|
||||
!NlpParser/testbench.txt
|
||||
!NlpParser/README.md
|
||||
NlpTr/*
|
||||
|
||||
.vscode/
|
||||
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Ww][Ii][Nn]32/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Ll]ogs/
|
||||
Temp/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUnit
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
nunit-*.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# ASP.NET Scaffolding
|
||||
ScaffoldingReadMe.txt
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Coverlet is a free, cross platform Code Coverage Tool
|
||||
coverage*[.json, .xml, .info]
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# NuGet Symbol Packages
|
||||
*.snupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- [Bb]ackup.rdl
|
||||
*- [Bb]ackup ([0-9]).rdl
|
||||
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
|
||||
2
LICENSE
2
LICENSE
@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2022-2024 yyc12345
|
||||
Copyright (c) 2022-2023 yyc12345
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
378
NlpCodec/.gitignore
vendored
378
NlpCodec/.gitignore
vendored
@ -1,378 +0,0 @@
|
||||
# -------------------- Custom --------------------
|
||||
# Disable install and build directory
|
||||
out/
|
||||
build/
|
||||
install/
|
||||
|
||||
# -------------------- CMake --------------------
|
||||
CMakeLists.txt.user
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
CMakeScripts
|
||||
Testing
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
install_manifest.txt
|
||||
compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
_deps
|
||||
|
||||
# -------------------- Visual Studio --------------------
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Ww][Ii][Nn]32/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Ll]ogs/
|
||||
Temp/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUnit
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
nunit-*.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# ASP.NET Scaffolding
|
||||
ScaffoldingReadMe.txt
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Coverlet is a free, cross platform Code Coverage Tool
|
||||
coverage*[.json, .xml, .info]
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# NuGet Symbol Packages
|
||||
*.snupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- [Bb]ackup.rdl
|
||||
*- [Bb]ackup ([0-9]).rdl
|
||||
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
@ -1,45 +0,0 @@
|
||||
cmake_minimum_required(VERSION 3.23)
|
||||
project(NlpCodec LANGUAGES CXX)
|
||||
|
||||
# Find ZLib packages
|
||||
find_package(ZLIB REQUIRED)
|
||||
|
||||
# Build executable
|
||||
add_executable(NlpCodec "")
|
||||
# Setup sources file and no need to setup headers
|
||||
target_sources(NlpCodec
|
||||
PRIVATE
|
||||
NlpCodec.cpp
|
||||
)
|
||||
# Link with ZLib
|
||||
target_link_libraries(NlpCodec
|
||||
PRIVATE
|
||||
${ZLIB_LIBRARIES}
|
||||
)
|
||||
# Setup standard
|
||||
set_target_properties(NlpCodec
|
||||
PROPERTIES
|
||||
CXX_STANDARD 20
|
||||
CXX_STANDARD_REQUIRED 20
|
||||
CXX_EXTENSION OFF
|
||||
)
|
||||
|
||||
# Extra options for MSVC
|
||||
# Unicode charset
|
||||
target_compile_definitions(NlpCodec
|
||||
PRIVATE
|
||||
$<$<CXX_COMPILER_ID:MSVC>:UNICODE>
|
||||
$<$<CXX_COMPILER_ID:MSVC>:_UNICODE>
|
||||
)
|
||||
# Order UTF-8 in both runtime and source environment
|
||||
target_compile_options(NlpCodec
|
||||
PRIVATE
|
||||
$<$<CXX_COMPILER_ID:MSVC>:/utf-8>
|
||||
)
|
||||
|
||||
# Install built artifact
|
||||
include(GNUInstallDirs)
|
||||
install(TARGETS NlpCodec
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
)
|
||||
|
||||
@ -1,366 +0,0 @@
|
||||
#include <zlib.h>
|
||||
#include <iostream>
|
||||
#include <cstdint>
|
||||
#include <cinttypes>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
|
||||
namespace NlpCodec {
|
||||
|
||||
class NlpException : public std::exception {
|
||||
public:
|
||||
NlpException(const char* msg) : message(msg ? msg : "") {}
|
||||
NlpException(const NlpException& rhs) : message(rhs.message) {}
|
||||
virtual ~NlpException() {}
|
||||
[[nodiscard]] virtual const char* what() const noexcept override { return message.c_str(); }
|
||||
private:
|
||||
std::string message;
|
||||
};
|
||||
|
||||
/// @brief The safe version of static_cast which throw exception
|
||||
/// if given value can not be cast into given type (out of range).
|
||||
template<typename _TyTo, typename _TyFrom>
|
||||
static constexpr _TyTo SafeCast(_TyFrom value) {
|
||||
if (!std::in_range<_TyTo>(value))
|
||||
throw NlpException(
|
||||
"Fail to cast integral number because given value is greater than container."
|
||||
"This is usually caused by your input or output file is too long.");
|
||||
return static_cast<_TyTo>(value);
|
||||
}
|
||||
|
||||
/// @brief The magic DWORD for file length encrption.
|
||||
/// @details It is actually the DWORD consisted by the first 4 bytes of XOR_ARRAY.
|
||||
constexpr const uint32_t MAGIC_DWORD = 0xF956A82Cu;
|
||||
constexpr const uint32_t CHECKSUM_OFFSET = 1072u;
|
||||
/// @brief The size of extra part of NLP file which store the size of original plain text file.
|
||||
constexpr const size_t TAIL_SIZE = sizeof(uint32_t) * 2u;
|
||||
|
||||
/// @brief The core array for data encryption.
|
||||
/// @details First byte will XOR with the first byte of this array, and so on.
|
||||
/// When reaching the tail of this array, next give byte will perform XOR with the first byte again and so on.
|
||||
constexpr const uint8_t XOR_ARRAY[] {
|
||||
0x2C, 0xA8, 0x56, 0xF9, 0xBD, 0xA6, 0x8D, 0x15, 0x25, 0x38, 0x1A, 0xD4, 0x65, 0x58, 0x28, 0x37,
|
||||
0xFA, 0x6B, 0xB5, 0xA1, 0x2C, 0x96, 0x13, 0xA2, 0xAB, 0x4F, 0xC5, 0xA1, 0x3E, 0xA7, 0x91, 0x8D,
|
||||
0x2C, 0xDF, 0x78, 0x6D, 0x3C, 0xFC, 0x92, 0x1F, 0x1A, 0x62, 0xA7, 0x9C, 0x92, 0x29, 0x44, 0x6D,
|
||||
0x3D, 0xA9, 0x2B, 0xE1, 0x91, 0xAD, 0x49, 0x3C, 0xE2, 0x33, 0xD2, 0x1A, 0x55, 0x92, 0xE7, 0x95,
|
||||
0x8C, 0xDA, 0xD2, 0xCD, 0xA2, 0xCF, 0x92, 0x9A, 0xE1, 0xF9, 0x3A, 0x26, 0xFA, 0xC4, 0xA9, 0x23,
|
||||
0xA9, 0x4D, 0x1A, 0x2C, 0x3C, 0x2A, 0xAC, 0x62, 0xA3, 0x92, 0xAC, 0x1F, 0x3E, 0xA6, 0xC9, 0xC8,
|
||||
0x63, 0xCA, 0x52, 0xF9, 0xFB, 0x3A, 0x9C, 0x2A, 0xB2, 0x1A, 0x8D, 0x9A, 0x8C, 0x2A, 0x9C, 0x32,
|
||||
0xAA, 0xC3, 0xA2, 0x97, 0x34, 0x92, 0xFA, 0x71, 0xBE, 0x3F, 0xAC, 0x28, 0x22, 0x9F, 0xAC, 0xE8
|
||||
};
|
||||
/// @brief The size of above array.
|
||||
constexpr const size_t XOR_ARRAY_LEN = sizeof(XOR_ARRAY) / sizeof(uint8_t);
|
||||
/// @brief A convenient mask for above array when performing modulo.
|
||||
constexpr const size_t XOR_ARRAY_MASK = XOR_ARRAY_LEN - 1u;
|
||||
// Use a static_assert to confirm computed XOR_ARRAY_MASK is what we desired.
|
||||
// Because some stupid programmers (like me) may change above array and fill a series of wrong data,
|
||||
// then this mask was computed wrongly.
|
||||
static_assert(XOR_ARRAY_MASK == 0x7Fu);
|
||||
|
||||
static void GeneralXorOperation(void* data, size_t data_len) {
|
||||
uint8_t* ptr = reinterpret_cast<uint8_t*>(data);
|
||||
for (size_t i = 0u; i < data_len; ++i) {
|
||||
ptr[i] ^= XOR_ARRAY[i & XOR_ARRAY_MASK];
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Get the length of given file stream.
|
||||
static uint32_t GetFileLength(std::ifstream& fin) {
|
||||
// Fetch the types this stream used for following convenience.
|
||||
using stream_pos_t = std::ifstream::pos_type;
|
||||
using stream_off_t = std::ifstream::off_type;
|
||||
|
||||
// Backups current file cursor.
|
||||
stream_pos_t current_pos = fin.tellg();
|
||||
// Seek to the tail and get corresponding offset to get the length of file.
|
||||
fin.seekg(0, std::ios_base::end);
|
||||
stream_pos_t tail_pos = fin.tellg();
|
||||
if (std::numeric_limits<uint32_t>::max() < tail_pos)
|
||||
throw NlpException("The size of given file is too large. It should not larger than the capacity of uint32_t.");
|
||||
// Restore to previous backup file cursor
|
||||
fin.seekg(static_cast<stream_off_t>(current_pos), std::ios_base::beg);
|
||||
|
||||
// Safely reurn cast length.
|
||||
return SafeCast<uint32_t>(tail_pos);
|
||||
}
|
||||
|
||||
// HINTS:
|
||||
// In zlib, uLong and uLongf is 32-bit or more.
|
||||
// So when casting them to uint32_t, you need use SafeCast to perform boundary check.
|
||||
// However, you can directly cast uint32_t to them because there is no overflow issue.
|
||||
// Additionally, uInt is 16-bit or more.
|
||||
// So when processing with uInt, please more carefully.
|
||||
|
||||
static void EncodeNlp(std::ifstream& fin, std::ofstream& fout) {
|
||||
// Get file length and fetch
|
||||
uint32_t raw_size = GetFileLength(fin);
|
||||
// Fetch corresponding zlib boundary for the convenience of zlib encode.
|
||||
// uLong is 32-bit or more, so we need check whether uint32_t can hold the result first.
|
||||
uint32_t computed_boundary = SafeCast<uint32_t>(compressBound(static_cast<uLong>(raw_size)));
|
||||
|
||||
// Create buffer first
|
||||
std::unique_ptr<char[]> inbuf(new(std::nothrow) char[raw_size]);
|
||||
std::unique_ptr<char[]> outbuf(new(std::nothrow) char[computed_boundary]);
|
||||
if (inbuf == nullptr || outbuf == nullptr)
|
||||
throw NlpException("Fail to allocate memory.");
|
||||
|
||||
// Read data from file to input buffer
|
||||
fin.read(inbuf.get(), raw_size);
|
||||
if (!fin.good() || fin.gcount() != raw_size)
|
||||
throw NlpException("Fail to read file data into buffer.");
|
||||
|
||||
// Do XOR operation
|
||||
GeneralXorOperation(inbuf.get(), raw_size);
|
||||
|
||||
// Do compress and get the size of compressed data.
|
||||
uLongf dest_len = static_cast<uLongf>(computed_boundary);
|
||||
int ret = compress2(
|
||||
reinterpret_cast<Bytef*>(outbuf.get()), &dest_len,
|
||||
reinterpret_cast<Bytef*>(inbuf.get()), static_cast<uLong>(raw_size),
|
||||
Z_BEST_COMPRESSION
|
||||
);
|
||||
// Check ZLib result.
|
||||
if (ret != Z_OK)
|
||||
throw NlpException("Zlib compress() failed.");
|
||||
// Fetch final compressed size.
|
||||
uint32_t compressed_size = SafeCast<uint32_t>(dest_len);
|
||||
|
||||
// Produce checksum
|
||||
uint32_t checksum = static_cast<uint32_t>(adler32(0u, reinterpret_cast<Bytef*>(outbuf.get()), SafeCast<uInt>(compressed_size)));
|
||||
|
||||
// Write compressed data into file
|
||||
fout.write(outbuf.get(), compressed_size);
|
||||
if (!fout.good())
|
||||
throw NlpException("Fail to write data into file.");
|
||||
|
||||
// Raw size and checksum need some extra encryption before writting
|
||||
raw_size = static_cast<uint32_t>(-(static_cast<int32_t>(raw_size) + 1)) ^ MAGIC_DWORD;
|
||||
checksum = checksum + CHECKSUM_OFFSET;
|
||||
|
||||
// Write raw size and checksum
|
||||
fout.write(reinterpret_cast<char*>(&raw_size), sizeof(uint32_t));
|
||||
if (!fout.good())
|
||||
throw NlpException("Fail to write raw size into file.");
|
||||
fout.write(reinterpret_cast<char*>(&checksum), sizeof(uint32_t));
|
||||
if (!fout.good())
|
||||
throw NlpException("Fail to write checksum into file.");
|
||||
|
||||
}
|
||||
|
||||
static void DecodeNlp(std::ifstream& fin, std::ofstream& fout) {
|
||||
// Seek to tail to get essential data
|
||||
uint32_t compressed_size = GetFileLength(fin);
|
||||
if (compressed_size < TAIL_SIZE)
|
||||
throw NlpException("Invalid file. File is too short.");
|
||||
|
||||
// Get expected raw size and checksum
|
||||
compressed_size -= TAIL_SIZE;
|
||||
fin.seekg(compressed_size, std::ios_base::beg);
|
||||
uint32_t expected_raw_size = 0u, expected_checksum = 0u;
|
||||
fin.read(reinterpret_cast<char*>(&expected_raw_size), sizeof(uint32_t));
|
||||
fin.read(reinterpret_cast<char*>(&expected_checksum), sizeof(uint32_t));
|
||||
fin.seekg(0, std::ios_base::beg);
|
||||
|
||||
// Raw size and checksum data need to do some extra decryption.
|
||||
expected_raw_size = static_cast<uint32_t>(-1 - static_cast<int32_t>(MAGIC_DWORD ^ expected_raw_size));
|
||||
expected_checksum = expected_checksum - CHECKSUM_OFFSET;
|
||||
|
||||
// Allocate memory to store data
|
||||
std::unique_ptr<char[]> inbuf(new(std::nothrow) char[compressed_size]);
|
||||
std::unique_ptr<char[]> outbuf(new(std::nothrow) char[expected_raw_size]);
|
||||
if (inbuf == nullptr || outbuf == nullptr)
|
||||
throw NlpException("Fail to allocate memory.");
|
||||
|
||||
// Read file into buffer
|
||||
fin.read(inbuf.get(), compressed_size);
|
||||
if (!fin.good() || fin.gcount() != compressed_size)
|
||||
throw NlpException("Fail to read data into buffer.\n");
|
||||
|
||||
// Test checksum
|
||||
uint32_t checksum = static_cast<uint32_t>(adler32(0u, reinterpret_cast<Bytef*>(inbuf.get()), SafeCast<uInt>(compressed_size)));
|
||||
if (checksum != expected_checksum) {
|
||||
fprintf(stdout, "[ERR] Fail to match crc32. Expect 0x%" PRIx32 " got 0x%" PRIx32 ".\n",
|
||||
expected_checksum, checksum
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Do decompress
|
||||
uLongf _destLen = static_cast<uLongf>(expected_raw_size);
|
||||
int ret = uncompress(
|
||||
reinterpret_cast<Bytef*>(outbuf.get()), &_destLen,
|
||||
reinterpret_cast<Bytef*>(inbuf.get()), static_cast<uLong>(compressed_size)
|
||||
);
|
||||
// Check zlib result
|
||||
if (ret != Z_OK)
|
||||
throw NlpException("Zlib uncompress() failed.");
|
||||
|
||||
// do xor operation
|
||||
GeneralXorOperation(outbuf.get(), expected_raw_size);
|
||||
|
||||
// Write result into file
|
||||
fout.write(outbuf.get(), expected_raw_size);
|
||||
if (!fout.good())
|
||||
throw NlpException("Fail to write data into file.");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace NlpCodec::Runtime {
|
||||
|
||||
enum class UserOperation {
|
||||
Encode,
|
||||
Decode,
|
||||
Version,
|
||||
Help
|
||||
};
|
||||
|
||||
struct UserRequest {
|
||||
UserOperation mUserOperation;
|
||||
std::filesystem::path mInputFile;
|
||||
std::filesystem::path mOutputFile;
|
||||
};
|
||||
|
||||
static void PrintHelp() {
|
||||
std::cout
|
||||
<< "NlpCodec Usage" << std::endl
|
||||
<< "NlpCodec [encode | decode | version | help] <src> <dest>" << std::endl
|
||||
<< std::endl
|
||||
<< "version - print version info about this program." << std::endl
|
||||
<< "help - print this page." << std::endl
|
||||
<< std::endl
|
||||
<< "encode - encode text file into NLP file." << std::endl
|
||||
<< "decode - decode NLP file into text file." << std::endl
|
||||
<< "<src> - the source file." << std::endl
|
||||
<< " the path to text file in encode mode." << std::endl
|
||||
<< " the path to NLP file in decode mode." << std::endl
|
||||
<< "<dest> - the destination file." << std::endl
|
||||
<< " the path to NLP file in encode mode." << std::endl
|
||||
<< " the path to text file in decode mode." << std::endl
|
||||
<< "" << std::endl;
|
||||
}
|
||||
|
||||
static void PrintVersion() {
|
||||
std::cout
|
||||
<< "NlpCodec built at " __DATE__ " " __TIME__ << std::endl
|
||||
<< "MIT License. Copyright (c) 2022-2024 yyc12345" << std::endl;
|
||||
}
|
||||
|
||||
static UserRequest ResolveArguments(int argc, char* argv[]) {
|
||||
// Prepare return value
|
||||
UserRequest ret { UserOperation::Version, "", "" };
|
||||
|
||||
switch (argc) {
|
||||
case 2: {
|
||||
// Get mode string
|
||||
std::string mode(argv[1]);
|
||||
|
||||
// Check `help` and `version`
|
||||
if (mode == "version") {
|
||||
ret.mUserOperation = UserOperation::Version;
|
||||
} else if (mode == "help") {
|
||||
ret.mUserOperation = UserOperation::Help;
|
||||
} else {
|
||||
// Not matched.
|
||||
throw NlpException("Invalid argument! Must be one of `version` or `help`");
|
||||
}
|
||||
|
||||
// Return value
|
||||
return ret;
|
||||
}
|
||||
case 4: {
|
||||
// Get mode string
|
||||
std::string mode(argv[1]);
|
||||
|
||||
// Check `encode` and `decode`
|
||||
if (mode == "encode") {
|
||||
ret.mUserOperation = UserOperation::Encode;
|
||||
} else if (mode == "decode") {
|
||||
ret.mUserOperation = UserOperation::Decode;
|
||||
} else {
|
||||
// Not matched.
|
||||
throw NlpException("Invalid argument! Must be one of `encode` or `decode`");
|
||||
}
|
||||
|
||||
// Setup input output file path
|
||||
ret.mInputFile = std::filesystem::path(argv[2]);
|
||||
ret.mOutputFile = std::filesystem::path(argv[3]);
|
||||
|
||||
// Return value
|
||||
return ret;
|
||||
}
|
||||
default:
|
||||
throw NlpException("Invalid argument count!");
|
||||
}
|
||||
}
|
||||
|
||||
static void ExecuteWorker(const UserRequest& user_request) {
|
||||
// Take action according to different request first
|
||||
bool is_encode;
|
||||
switch (user_request.mUserOperation) {
|
||||
case UserOperation::Version:
|
||||
PrintVersion();
|
||||
return;
|
||||
case NlpCodec::Runtime::UserOperation::Help:
|
||||
PrintHelp();
|
||||
return;
|
||||
case NlpCodec::Runtime::UserOperation::Encode:
|
||||
is_encode = true;
|
||||
break;
|
||||
case NlpCodec::Runtime::UserOperation::Decode:
|
||||
is_encode = false;
|
||||
break;
|
||||
}
|
||||
|
||||
// Do real codec related works.
|
||||
// Try to open files
|
||||
std::ifstream in_file;
|
||||
in_file.open(user_request.mInputFile, std::ios_base::in | std::ios_base::binary);
|
||||
std::ofstream out_file;
|
||||
out_file.open(user_request.mOutputFile, std::ios_base::out | std::ios_base::binary);
|
||||
// Check file status
|
||||
if (!in_file.is_open() || !out_file.is_open()) {
|
||||
throw NlpException("Fail to open input or output file.");
|
||||
}
|
||||
|
||||
// Perform codec
|
||||
if (is_encode) {
|
||||
::NlpCodec::EncodeNlp(in_file, out_file);
|
||||
} else {
|
||||
::NlpCodec::DecodeNlp(in_file, out_file);
|
||||
}
|
||||
|
||||
// Free resources
|
||||
in_file.close();
|
||||
out_file.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
|
||||
// Try parsing given arguments
|
||||
NlpCodec::Runtime::UserRequest user_request;
|
||||
try {
|
||||
user_request = NlpCodec::Runtime::ResolveArguments(argc, argv);
|
||||
} catch (const NlpCodec::NlpException& e) {
|
||||
std::cerr << "[Argument Error] " << e.what() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Try executing real wroker
|
||||
try {
|
||||
NlpCodec::Runtime::ExecuteWorker(user_request);
|
||||
} catch (const NlpCodec::NlpException& e) {
|
||||
std::cerr << "[Codec Error] " << e.what() << std::endl;
|
||||
return 2;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
# Nlp Codec
|
||||
|
||||
## Requirements
|
||||
|
||||
* C++ 17 standard libs.
|
||||
* zlib
|
||||
|
||||
## Linux
|
||||
|
||||
```
|
||||
mkdir out
|
||||
cd out
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
|
||||
## Windows MSYS2
|
||||
|
||||
```
|
||||
mkdir out
|
||||
cd out
|
||||
cmake -G "Unix Makefiles" ..
|
||||
make
|
||||
```
|
||||
19
NlpEncoder/CMakeLists.txt
Normal file
19
NlpEncoder/CMakeLists.txt
Normal file
@ -0,0 +1,19 @@
|
||||
cmake_minimum_required(VERSION 3.12)
|
||||
project(NlpEncoder LANGUAGES CXX)
|
||||
|
||||
# find packages
|
||||
find_package(ZLIB REQUIRED)
|
||||
|
||||
# set standard
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
|
||||
# generate program
|
||||
add_executable(NlpEncoder NlpEncoder.cpp)
|
||||
target_link_libraries(NlpEncoder
|
||||
PRIVATE
|
||||
${ZLIB_LIBRARIES}
|
||||
)
|
||||
target_include_directories(NlpEncoder
|
||||
PRIVATE
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
)
|
||||
233
NlpEncoder/NlpEncoder.cpp
Normal file
233
NlpEncoder/NlpEncoder.cpp
Normal file
@ -0,0 +1,233 @@
|
||||
#include <zlib.h>
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cinttypes>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
|
||||
namespace NlpEncoder {
|
||||
|
||||
constexpr const uint8_t g_XorArray[] {
|
||||
0x2C, 0xA8, 0x56, 0xF9, 0xBD, 0xA6, 0x8D, 0x15, 0x25, 0x38, 0x1A, 0xD4, 0x65, 0x58, 0x28, 0x37,
|
||||
0xFA, 0x6B, 0xB5, 0xA1, 0x2C, 0x96, 0x13, 0xA2, 0xAB, 0x4F, 0xC5, 0xA1, 0x3E, 0xA7, 0x91, 0x8D,
|
||||
0x2C, 0xDF, 0x78, 0x6D, 0x3C, 0xFC, 0x92, 0x1F, 0x1A, 0x62, 0xA7, 0x9C, 0x92, 0x29, 0x44, 0x6D,
|
||||
0x3D, 0xA9, 0x2B, 0xE1, 0x91, 0xAD, 0x49, 0x3C, 0xE2, 0x33, 0xD2, 0x1A, 0x55, 0x92, 0xE7, 0x95,
|
||||
0x8C, 0xDA, 0xD2, 0xCD, 0xA2, 0xCF, 0x92, 0x9A, 0xE1, 0xF9, 0x3A, 0x26, 0xFA, 0xC4, 0xA9, 0x23,
|
||||
0xA9, 0x4D, 0x1A, 0x2C, 0x3C, 0x2A, 0xAC, 0x62, 0xA3, 0x92, 0xAC, 0x1F, 0x3E, 0xA6, 0xC9, 0xC8,
|
||||
0x63, 0xCA, 0x52, 0xF9, 0xFB, 0x3A, 0x9C, 0x2A, 0xB2, 0x1A, 0x8D, 0x9A, 0x8C, 0x2A, 0x9C, 0x32,
|
||||
0xAA, 0xC3, 0xA2, 0x97, 0x34, 0x92, 0xFA, 0x71, 0xBE, 0x3F, 0xAC, 0x28, 0x22, 0x9F, 0xAC, 0xE8
|
||||
};
|
||||
constexpr const size_t g_XorArrayLen = sizeof(g_XorArray) / sizeof(uint8_t);
|
||||
constexpr const uint32_t MAGIC_DWORD = 0xF956A82Cu;
|
||||
constexpr const size_t TAIL_SIZE = sizeof(uint32_t) * 2u;
|
||||
|
||||
void GeneralXorOperation(void* data, size_t datalen) {
|
||||
uint8_t* ptr = reinterpret_cast<uint8_t*>(data);
|
||||
for (size_t i = 0u; i < datalen; ++i) {
|
||||
ptr[i] ^= g_XorArray[i & 0x7Fu];
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t GetFileLength(std::ifstream& fin) {
|
||||
// backup
|
||||
uint64_t curpos = static_cast<uint64_t>(fin.tellg());
|
||||
// get tail
|
||||
fin.seekg(0, std::ios_base::end);
|
||||
uint32_t tail = static_cast<uint32_t>(fin.tellg());
|
||||
// restore
|
||||
fin.seekg(static_cast<std::ifstream::off_type>(curpos), std::ios_base::beg);
|
||||
|
||||
return tail;
|
||||
}
|
||||
|
||||
bool EncodeNlp(std::ifstream& fin, std::ofstream& fout) {
|
||||
// get file length and decide zlib boundary
|
||||
uint32_t rawsize = GetFileLength(fin);
|
||||
uint32_t compboundary = static_cast<uint32_t>(compressBound(static_cast<uLong>(rawsize)));
|
||||
|
||||
// create buffer first
|
||||
std::unique_ptr<char[]> inbuf(new(std::nothrow) char[rawsize]);
|
||||
std::unique_ptr<char[]> outbuf(new(std::nothrow) char[compboundary]);
|
||||
if (inbuf == nullptr || outbuf == nullptr) {
|
||||
fputs("[ERR] Fail to allocate memory.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// read data from file
|
||||
fin.read(inbuf.get(), rawsize);
|
||||
if (!fin.good() || fin.gcount() != rawsize) {
|
||||
fputs("[ERR] Fail to read data into buffer.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// do xor operation
|
||||
GeneralXorOperation(inbuf.get(), rawsize);
|
||||
|
||||
// do compress and get the size of compressed data
|
||||
uLongf _destLen = static_cast<uLongf>(compboundary);
|
||||
int ret = compress2(
|
||||
reinterpret_cast<Bytef*>(outbuf.get()), &_destLen,
|
||||
reinterpret_cast<Bytef*>(inbuf.get()), rawsize,
|
||||
Z_BEST_COMPRESSION
|
||||
);
|
||||
if (ret != Z_OK) {
|
||||
fputs("[ERR] Zlib compress() failed.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
uint32_t compsize = static_cast<uint32_t>(_destLen);
|
||||
|
||||
// produce checksum
|
||||
uint32_t checksum = static_cast<uint32_t>(adler32(0u, reinterpret_cast<Bytef*>(outbuf.get()), static_cast<uInt>(compsize)));
|
||||
|
||||
// write compressed data into file
|
||||
fout.write(outbuf.get(), compsize);
|
||||
if (!fout.good()) {
|
||||
fputs("[ERR] Fail to write data into file.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// raw size and checksum need some extra operation before writting
|
||||
rawsize = static_cast<uint32_t>(-(static_cast<int32_t>(rawsize) + 1)) ^ MAGIC_DWORD;
|
||||
checksum = checksum + 1072u;
|
||||
|
||||
// write raw size and checksum
|
||||
fout.write(reinterpret_cast<char*>(&rawsize), sizeof(uint32_t));
|
||||
if (!fout.good()) {
|
||||
fputs("[ERR] Fail to write raw size into file.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
fout.write(reinterpret_cast<char*>(&checksum), sizeof(uint32_t));
|
||||
if (!fout.good()) {
|
||||
fputs("[ERR] Fail to write checksum into file.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DecodeNlp(std::ifstream& fin, std::ofstream& fout) {
|
||||
// seek to tail to get essential data
|
||||
uint32_t compsize = GetFileLength(fin);
|
||||
if (compsize < TAIL_SIZE) {
|
||||
fputs("[ERR] Invalid file.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
compsize -= TAIL_SIZE;
|
||||
fin.seekg(compsize, std::ios_base::beg);
|
||||
uint32_t expected_rawlen = 0u, expected_checksum = 0u;
|
||||
fin.read(reinterpret_cast<char*>(&expected_rawlen), sizeof(uint32_t));
|
||||
fin.read(reinterpret_cast<char*>(&expected_checksum), sizeof(uint32_t));
|
||||
fin.seekg(0, std::ios_base::beg);
|
||||
|
||||
// these tail data need to do some processes
|
||||
expected_rawlen = static_cast<uint32_t>(-1 - static_cast<int32_t>(MAGIC_DWORD ^ expected_rawlen));
|
||||
expected_checksum = expected_checksum - 1072u;
|
||||
|
||||
// allocate memory to store data
|
||||
std::unique_ptr<char[]> inbuf(new(std::nothrow) char[compsize]);
|
||||
std::unique_ptr<char[]> outbuf(new(std::nothrow) char[expected_rawlen]);
|
||||
if (inbuf == nullptr || outbuf == nullptr) {
|
||||
fputs("[ERR] Fail to allocate memory.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// read into buffer
|
||||
fin.read(inbuf.get(), compsize);
|
||||
if (!fin.good() || fin.gcount() != compsize) {
|
||||
fputs("[ERR] Fail to read data into buffer.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// test checksum
|
||||
uint32_t checksum = static_cast<uint32_t>(adler32(0u, reinterpret_cast<Bytef*>(inbuf.get()), static_cast<uInt>(compsize)));
|
||||
if (checksum != expected_checksum) {
|
||||
fprintf(stdout, "[ERR] Fail to match crc32. Expect 0x%" PRIx32 " got 0x%" PRIx32 ".\n",
|
||||
expected_checksum, checksum
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// do uncompress
|
||||
uLongf _destLen = static_cast<uLongf>(expected_rawlen);
|
||||
int ret = uncompress(
|
||||
reinterpret_cast<Bytef*>(outbuf.get()), &_destLen,
|
||||
reinterpret_cast<Bytef*>(inbuf.get()), static_cast<uLong>(compsize)
|
||||
);
|
||||
if (ret != Z_OK) {
|
||||
fputs("[ERR] Zlib uncompress() failed.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
// do xor operation
|
||||
GeneralXorOperation(outbuf.get(), expected_rawlen);
|
||||
|
||||
// write into file
|
||||
fout.write(outbuf.get(), expected_rawlen);
|
||||
if (!fout.good()) {
|
||||
fputs("[ERR] Fail to write data into file.\n", stdout);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static void PrintHelp(void) {
|
||||
fputs("NlpEncoder Usage\n", stdout);
|
||||
fputs("\n", stdout);
|
||||
fputs("NlpEncoder [compress | uncompress] <src> <dest>\n", stdout);
|
||||
fputs("compress - compress text file into nlp file.\n", stdout);
|
||||
fputs("uncompress - decompress nlp file into text file.\n", stdout);
|
||||
fputs("<src> - the source file. text file in compress mode. nlp file in uncompress mode.\n", stdout);
|
||||
fputs("<dest> - the destination file. nlp file in compress mode. text file in uncompress mode.\n", stdout);
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
|
||||
// check arguments
|
||||
if (argc != 4) {
|
||||
fputs("[ERR] Invalid arguments!\n", stdout);
|
||||
PrintHelp();
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::string mode(argv[1]);
|
||||
if (mode != "compress" && mode != "uncompress") {
|
||||
fputs("[ERR] Unknow operation!\n", stdout);
|
||||
PrintHelp();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// try initializing files
|
||||
std::ifstream infile;
|
||||
infile.open(std::filesystem::path(argv[2]), std::ios_base::in | std::ios_base::binary);
|
||||
std::ofstream outfile;
|
||||
outfile.open(std::filesystem::path(argv[3]), std::ios_base::out | std::ios_base::binary);
|
||||
|
||||
if (!infile.is_open() || !outfile.is_open()) {
|
||||
fputs("[ERR] Fail to open file!\n", stdout);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// do real work
|
||||
bool result = true;
|
||||
if (mode == "compress") {
|
||||
result = NlpEncoder::EncodeNlp(infile, outfile);
|
||||
} else {
|
||||
result = NlpEncoder::DecodeNlp(infile, outfile);
|
||||
}
|
||||
|
||||
// free resources and report
|
||||
infile.close();
|
||||
outfile.close();
|
||||
|
||||
if (!result) {
|
||||
fputs("[ERR] Encoder failed!\n", stdout);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
3
NlpEncoder/README.md
Normal file
3
NlpEncoder/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
# Nlp Encoder
|
||||
|
||||
Requirements: C++ 17 standard.
|
||||
42
NlpParser/.gitignore
vendored
42
NlpParser/.gitignore
vendored
@ -1,42 +0,0 @@
|
||||
# ===== Custom =====
|
||||
# Only preserve testbench in assets.
|
||||
assets/*
|
||||
!assets/testbench.txt
|
||||
|
||||
# ===== ANTLR Output =====
|
||||
*.interp
|
||||
*.tokens
|
||||
|
||||
Nlp*.java
|
||||
|
||||
# ===== Eclipse Java =====
|
||||
# Eclipse projects
|
||||
.classpath
|
||||
.project
|
||||
.settings/
|
||||
.metadata
|
||||
|
||||
# Compiled class file
|
||||
*.class
|
||||
|
||||
# Log file
|
||||
*.log
|
||||
|
||||
# BlueJ files
|
||||
*.ctxt
|
||||
|
||||
# Mobile Tools for Java (J2ME)
|
||||
.mtj.tmp/
|
||||
|
||||
# Package Files #
|
||||
*.jar
|
||||
*.war
|
||||
*.nar
|
||||
*.ear
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
|
||||
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||
hs_err_pid*
|
||||
replay_pid*
|
||||
@ -19,12 +19,11 @@ import java.io.FileOutputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
public class MainRunner {
|
||||
public class NlpRunner {
|
||||
public static class NlpJsonConverter extends NlpBaseListener {
|
||||
public NlpJsonConverter() {
|
||||
mGsonInstance = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create();
|
||||
mGsonInstance = new GsonBuilder().setPrettyPrinting().create();
|
||||
mRoot = new JsonObject();
|
||||
mSection = new JsonArray();
|
||||
mSectionStack = new Stack<JsonArray>();
|
||||
@ -148,7 +147,7 @@ public class MainRunner {
|
||||
}
|
||||
|
||||
private static void printHelp() {
|
||||
System.out.println("NlpParser <src> <dest>");
|
||||
System.out.println("NlpRunner <src> <dest>");
|
||||
System.out.println();
|
||||
System.out.println("<src> - the decoded nlp text file.");
|
||||
System.out.println("<dest> - the output json file.");
|
||||
@ -178,7 +177,7 @@ public class MainRunner {
|
||||
}
|
||||
|
||||
// start lex and parse
|
||||
CharStream input = CharStreams.fromStream(fin, Charset.forName("windows-1252"));
|
||||
CharStream input = CharStreams.fromStream(fin, StandardCharsets.UTF_8);
|
||||
NlpLexer lexer = new NlpLexer(input);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
NlpParser parser = new NlpParser(tokens);
|
||||
@ -8,9 +8,9 @@ Requirements:
|
||||
Useful commands:
|
||||
|
||||
* `antlr4 Nlp.g4`: Analyze Antlr format file.
|
||||
* `javac *.java`: Compile executable application.
|
||||
* `javac Nlp*.java`: Compile executable application.
|
||||
* Do testbench
|
||||
- `grun Nlp document -tree < testbench.txt`
|
||||
- `grun Nlp document -gui < testbench.txt`
|
||||
- `java MainRunner testbench.txt out/result.json`
|
||||
- `java NlpRunner testbench.txt result.json`
|
||||
|
||||
|
||||
164
NlpProc/.gitignore
vendored
164
NlpProc/.gitignore
vendored
@ -1,164 +0,0 @@
|
||||
## ---------- VS Code ----------
|
||||
.vscode/
|
||||
|
||||
## ---------- Python ----------
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
@ -2,18 +2,11 @@ import NlpUtils
|
||||
import jsondiff
|
||||
import collections
|
||||
|
||||
if NlpUtils.g_EnableDebugging:
|
||||
g_SupportedEncoding = {
|
||||
'template': ('English', ('windows-1252', ), )
|
||||
}
|
||||
else:
|
||||
g_SupportedEncoding = {
|
||||
'zh-cn': ('Chinese', ('utf-8', 'gb2312', ), )
|
||||
}
|
||||
g_SupportedEncoding = {
|
||||
'zh-cn': ('Chinese', ('utf-8', 'gb2312', ), )
|
||||
}
|
||||
|
||||
VtTrDataTuple = collections.namedtuple('VtTrDataTuple', ('rawNlp', 'trTemplate', 'trDiff', 'trIndex'))
|
||||
def GetNlpJsonPath(ver: str, lang: str) -> str:
|
||||
return f'../NlpTr/out/VT{ver}.{lang}.json'
|
||||
def GetRawNlpPath(ver: str, lang: str, enc: str) -> str:
|
||||
return f'../NlpTr/out/VT{ver}.{lang}.{enc}.txt'
|
||||
def GetTrPath(ver: str, lang: str) -> str:
|
||||
@ -23,39 +16,6 @@ def GetTrDiffPath(ver: str) -> str:
|
||||
def GetTrIndexPath(ver: str) -> str:
|
||||
return f'../NlpTr/VT{ver}.index'
|
||||
|
||||
g_CriticalFields: dict[str, str] = {
|
||||
'Common/Registry/0': 'Software\\\\Virtools\\\\Global',
|
||||
'Common/Registry/1': 'Usage Count',
|
||||
'Common/Timebomb/0': 'Key1',
|
||||
'Common/Timebomb/1': 'Key2',
|
||||
'Common/Timebomb/2': 'Key3',
|
||||
'Common/Timebomb/3': 'SYSINFO.SysInfo32\\\\CLSID',
|
||||
'Common/Timebomb/4': '\\\\csrsrv32.dll',
|
||||
'3D Layout/Registry/0': 'Software\\\\NeMo\\\\3D Layout',
|
||||
}
|
||||
def CriticalFieldChecker(nlpJson: dict):
|
||||
corrected: bool = False
|
||||
for k, v in g_CriticalFields.items():
|
||||
# analyze path and find the node
|
||||
path = k.split('/')
|
||||
assert path[-1].isdecimal()
|
||||
path_terminal = int(path[-1])
|
||||
path = path[:-1]
|
||||
|
||||
node = nlpJson
|
||||
for pathpart in path:
|
||||
node = node['key_map'][pathpart]
|
||||
|
||||
# check it
|
||||
if node['entries'][path_terminal] != v:
|
||||
# if not matched. correct it
|
||||
node['entries'][path_terminal] = v
|
||||
# and notify it
|
||||
corrected = True
|
||||
|
||||
if corrected:
|
||||
print('Some critical filed was changed in tr by accident. We have corrected them, but please check tr carefully')
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# load each version's diff data and patch data for conventient using
|
||||
@ -69,16 +29,15 @@ if __name__ == "__main__":
|
||||
preLoadedData[ver] = PreLoadedDiffIdxTuple._make((insertedKey, deletedKey, plainKeys))
|
||||
|
||||
# iterate lang first
|
||||
# because we use progressive patch. we need iterate vt ver in order for each single languages
|
||||
# because we use progressive patch. we need iterate vt ver in order
|
||||
for lang in NlpUtils.g_SupportedLangs:
|
||||
|
||||
prevPlainValues: list[str] = None
|
||||
for ver in NlpUtils.g_VirtoolsVersion:
|
||||
print(f'Loading {ver}.{lang}...')
|
||||
print(f'Processing {ver}.{lang}...')
|
||||
|
||||
# pick data from pre-loaded dict
|
||||
diffIdxData = preLoadedData[ver]
|
||||
plainKeys = diffIdxData.plainKeys
|
||||
|
||||
# load lang file
|
||||
# and only keeps its value.
|
||||
@ -99,12 +58,6 @@ if __name__ == "__main__":
|
||||
|
||||
# convert plain json to nlp json
|
||||
nlpJson = NlpUtils.PlainJson2NlpJson(plainKeys, plainValues)
|
||||
# check some critical fields
|
||||
CriticalFieldChecker(nlpJson)
|
||||
|
||||
if NlpUtils.g_EnableDebugging:
|
||||
NlpUtils.RemoveKeyMapInGeneratedNlpJson(nlpJson)
|
||||
NlpUtils.DumpJson(GetNlpJsonPath(ver, lang), nlpJson)
|
||||
|
||||
# write into file with different encoding
|
||||
lang_macro, encs = g_SupportedEncoding[lang]
|
||||
|
||||
@ -4,30 +4,16 @@ import io
|
||||
import json
|
||||
import re
|
||||
|
||||
g_EnableDebugging = False
|
||||
|
||||
g_VirtoolsVersion: tuple[str] = (
|
||||
'25', '30', '35', '40', '50',
|
||||
'25', '35', '40', '50',
|
||||
)
|
||||
g_SupportedLangs: tuple[str] = (
|
||||
'zh-cn',
|
||||
)
|
||||
|
||||
if g_EnableDebugging:
|
||||
g_SupportedLangs: tuple[str] = (
|
||||
'template',
|
||||
)
|
||||
else:
|
||||
g_SupportedLangs: tuple[str] = (
|
||||
'zh-cn',
|
||||
)
|
||||
|
||||
# ========== Basic File RW Functions ==========
|
||||
|
||||
def DumpJson(filepath: str, jsonData: dict):
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(jsonData, f,
|
||||
indent=2,
|
||||
sort_keys=False,
|
||||
ensure_ascii=False
|
||||
)
|
||||
json.dump(jsonData, f, indent=4, sort_keys=False)
|
||||
|
||||
def LoadJson(filepath: str) -> dict:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
@ -136,36 +122,24 @@ def InternalNlpJson2PlainJson(nlpJson: dict, stack: collections.deque, keyList:
|
||||
InternalNlpJson2PlainJson(entry, stack, keyList, valueList)
|
||||
stack.pop()
|
||||
|
||||
# ========== Json Converter ==========
|
||||
|
||||
def PlainJson2NlpJson(keyList: list[str], valueList: list[str]) -> dict:
|
||||
# create the base section
|
||||
# each section will have 3 k-v pair. language/section and entries are existed in original nlp json
|
||||
# each section will have 3 k-v pair. language/section and entities are existed in original nlp json
|
||||
# and key_map is served for path finding and convenient for looking for sub section.
|
||||
result: dict = {
|
||||
"language": "English",
|
||||
"entries": [],
|
||||
"entities": [],
|
||||
"key_map": {}
|
||||
}
|
||||
# inerate list and construct dict
|
||||
for k, v in zip(keyList, valueList):
|
||||
InternalPlainJson2NlpJson(result, k, v)
|
||||
return result
|
||||
def RemoveKeyMapInGeneratedNlpJson(nlpJson: dict) -> dict:
|
||||
# remove useless key map
|
||||
InternalDelNlpJsonKeyMap(nlpJson)
|
||||
return nlpJson
|
||||
def InternalDelNlpJsonKeyMap(nlpJson: dict):
|
||||
# recursively calling self
|
||||
for v in nlpJson['key_map'].values():
|
||||
InternalDelNlpJsonKeyMap(v)
|
||||
# then delete self
|
||||
del nlpJson['key_map']
|
||||
def InternalPlainJson2NlpJson(nlpJson: dict, pairKey: str, pairVal: str):
|
||||
keypath = pairKey.split('/')
|
||||
# confirm last node is number and remove it
|
||||
assert keypath[-1].isdecimal()
|
||||
keypath = keypath[:-1]
|
||||
keypath = keypath[0:-1]
|
||||
|
||||
# move to correct sub section
|
||||
for pathpart in keypath:
|
||||
@ -176,21 +150,21 @@ def InternalPlainJson2NlpJson(nlpJson: dict, pairKey: str, pairVal: str):
|
||||
# create a new one
|
||||
sub_section = {
|
||||
'section': pathpart,
|
||||
'entries': [],
|
||||
'entities': [],
|
||||
'key_map': {}
|
||||
}
|
||||
|
||||
# add into current section
|
||||
nlpJson['entries'].append(sub_section)
|
||||
nlpJson['entities'].append(sub_section)
|
||||
nlpJson['key_map'][pathpart] = sub_section
|
||||
|
||||
# move to the new created sub section
|
||||
nlpJson = sub_section
|
||||
|
||||
# insert data
|
||||
nlpJson['entries'].append(pairVal)
|
||||
nlpJson['entities'].append(pairVal)
|
||||
|
||||
|
||||
# ========== Raw Nlp Text Writer ==========
|
||||
|
||||
def DumpNlpJson(filepath: str, encoding: str, lang_macro: str, nlpJson: dict):
|
||||
# write in wb mode because we need explicitly write \r\n, not \n
|
||||
@ -198,16 +172,16 @@ def DumpNlpJson(filepath: str, encoding: str, lang_macro: str, nlpJson: dict):
|
||||
f.write(f'Language:{lang_macro}\r\n'.encode(encoding, errors='ignore'))
|
||||
InternalDumpNlpJson(f, encoding, 0, nlpJson)
|
||||
|
||||
# g_NlpJsonStrRepl1 = re.compile('\\\\')
|
||||
g_NlpJsonStrRepl1 = re.compile('\\\\')
|
||||
g_NlpJsonStrRepl2 = re.compile('\"')
|
||||
def NlpJsonStringProcessor(strl: str) -> str:
|
||||
return g_NlpJsonStrRepl2.sub('\"\"', strl)
|
||||
|
||||
def InternalDumpNlpJson(f: io.BufferedWriter, encoding: str, depth: int, nlpJson: dict):
|
||||
assert 'entries' in nlpJson
|
||||
assert 'entities' in nlpJson
|
||||
|
||||
is_first: bool = True
|
||||
for entity in nlpJson['entries']:
|
||||
for entity in nlpJson['entities']:
|
||||
if isinstance(entity, str):
|
||||
# write comma if not the first element
|
||||
if not is_first: f.write(','.encode(encoding))
|
||||
|
||||
@ -3,6 +3,3 @@
|
||||
Example:
|
||||
|
||||
Create templates: `py NlpJsonDecoder.py`
|
||||
Compile translations: `py NlpJsonEncoder.py`
|
||||
|
||||
NOTE: all python files should be executed in this folder. NOT ROOT folder.
|
||||
|
||||
@ -1,12 +1,6 @@
|
||||
# Nlp Src
|
||||
|
||||
This directory stores the vanilla NLP files copied from installed Virtools.
|
||||
Useful comparing differences command:
|
||||
|
||||
This directory may contains some processed temporary files if you are building this repository. These files are ignored by git so don't worry about them.
|
||||
|
||||
## Useful Commands
|
||||
|
||||
Compare plain text NLP file between different Virtools version when you have decompressed and decrypted vanilla NLP files.
|
||||
|
||||
`diff -u --strip-trailing-cr VT25.json VT50.json`
|
||||
`diff -u --strip-trailing-cr VT25.txt VT50.txt`
|
||||
`diff -u VT25.json VT50.json`
|
||||
`diff -u VT25.txt VT50.txt`
|
||||
|
||||
BIN
NlpSrc/VT30.nlp
BIN
NlpSrc/VT30.nlp
Binary file not shown.
2
NlpTr/.gitignore
vendored
2
NlpTr/.gitignore
vendored
@ -1,2 +0,0 @@
|
||||
# Ignore result
|
||||
out/
|
||||
@ -1,21 +0,0 @@
|
||||
# Nlp Translation
|
||||
|
||||
## Create New Language Translation
|
||||
|
||||
These parts almost are done by repository maintainer. This is just a manual when maintainer goes.
|
||||
|
||||
0. First, decide your preferred language macro. I take "zh-cn" in there for example.
|
||||
0. Executing `./Scripts/create_new_tr.sh "zh-cn"` in **ROOT** folder.
|
||||
0. You will get a bunch of language json file named like "VT25.zh-cn.json" and listed in NlpTr folder.
|
||||
0. Then navigate to NlpProc folder and register your language macro.
|
||||
- `NlpUtils.py`: Add your language macro as a entry in the second declaration of `g_SupportedLangs`.
|
||||
- `NlpJsonEncoder.py`: Navigate to the second declaration of `g_SupportedEncoding` In Add your macro, specify the language name shown in Virtools and, a tuple of common encoding of your language. Please note that UTF8 encoding is necessary for each language.
|
||||
|
||||
## How to Edit Translation
|
||||
|
||||
Choose your preferred translator, and fill the json correctly. I use Poedit anyway.
|
||||
|
||||
## How to Compile Translation
|
||||
|
||||
0. Executing `./Scripts/compile_tr.sh` in **ROOT** folder.
|
||||
0. Then all nlp files should be generated in NlpTr/out
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,23 +0,0 @@
|
||||
{
|
||||
"Schematic/Popup/11": "Sort By Activity",
|
||||
"3D Object Setup/Buttons/3": "Use Hardware Skinning",
|
||||
"3D Object Setup/Buttons/4": "Bones Mode",
|
||||
"3D Object Setup/Bones Mode/0": "Local",
|
||||
"3D Object Setup/Bones Mode/1": "World",
|
||||
"3D Object Setup/Bones Mode/2": "World View",
|
||||
"3D Object Setup/Bones Mode/3": "World View Proj",
|
||||
"3D Object Setup/Bones Info/0": "Bones Count",
|
||||
"3D Object Setup/Bones Info/1": "Max. Bones per Vertex",
|
||||
"Mesh Setup/Buttons/20": "Material / Channel Type",
|
||||
"Mesh Setup/Popup Menu/3": "Fill With...",
|
||||
"Hierarchy View/Errors/3": "A place can only be a 3D root!",
|
||||
"TextureType/0": "2d",
|
||||
"TextureType/1": "Cube",
|
||||
"TextureType/2": "Volume",
|
||||
"Channel Extended/0": "Add Extended Channel",
|
||||
"Channel Extended/1": "Float 1",
|
||||
"Channel Extended/2": "Float 2",
|
||||
"Channel Extended/3": "Float 3",
|
||||
"Channel Extended/4": "Float 4",
|
||||
"Channel Extended/5": "Channel Type..."
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"Schematic/Popup/11": "按活动排序",
|
||||
"3D Object Setup/Buttons/3": "使用硬件蒙皮",
|
||||
"3D Object Setup/Buttons/4": "骨骼模式",
|
||||
"3D Object Setup/Bones Mode/0": "本地",
|
||||
"3D Object Setup/Bones Mode/1": "世界",
|
||||
"3D Object Setup/Bones Mode/2": "世界视图",
|
||||
"3D Object Setup/Bones Mode/3": "世界视图投影",
|
||||
"3D Object Setup/Bones Info/0": "骨骼数量",
|
||||
"3D Object Setup/Bones Info/1": "每个顶点的最大骨骼数",
|
||||
"Mesh Setup/Buttons/20": "材质 / 通道类型",
|
||||
"Mesh Setup/Popup Menu/3": "填充...",
|
||||
"Hierarchy View/Errors/3": "一个位置只能是一个3D根!",
|
||||
"TextureType/0": "2d",
|
||||
"TextureType/1": "Cube",
|
||||
"TextureType/2": "Volume",
|
||||
"Channel Extended/0": "添加扩展通道",
|
||||
"Channel Extended/1": "浮点数1",
|
||||
"Channel Extended/2": "浮点数2",
|
||||
"Channel Extended/3": "浮点数3",
|
||||
"Channel Extended/4": "浮点数4",
|
||||
"Channel Extended/5": "通道类型..."
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
{
|
||||
"CK/Class Names/21": "Video",
|
||||
"CK/Class Names 2/21": "Videos",
|
||||
"Main Menu/33": "&Refresh Windows\\tF5",
|
||||
"Main Menu/44": "E&xport File",
|
||||
"Main Menu/45": "&Level",
|
||||
"Main Menu/46": "&Selection",
|
||||
"Schematic/Dialog/17": "source of parameter <%s> (owner object : <%s>) is not in schematic\\nDo you want to show the setup of <%s> ?",
|
||||
"Schematic/Dialog/18": "source of parameter <%s> (no owner object) is not in schematic\\n",
|
||||
"Schematic/Popup/27": "Move'n'Scroll Mode\\tG",
|
||||
"3D Object Setup/Bones Mode/4": "Inv Local",
|
||||
"3D Object Setup/Bones Mode/5": "Inv World",
|
||||
"3D Object Setup/Bones Mode/6": "Inv WorldView",
|
||||
"3D Object Setup/Bones Mode/7": "Inv World View Proj",
|
||||
"2D Sprite Setup/Popup Menu/6": "Save slot..."
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
{
|
||||
"CK/Class Names/21": "视频",
|
||||
"CK/Class Names 2/21": "视频",
|
||||
"Main Menu/33": "刷新窗口\\tF5",
|
||||
"Main Menu/44": "导出文件",
|
||||
"Main Menu/45": "层级",
|
||||
"Main Menu/46": "选择",
|
||||
"Schematic/Dialog/17": "参数<%s>的源(拥有者对象:<%s>)不在原理图中\\n您要显示<%s> 的设置吗?",
|
||||
"Schematic/Dialog/18": "参数<%s>的源(无拥有者对象)不在原理图中\\n",
|
||||
"Schematic/Popup/27": "移动滚动模式\\tG",
|
||||
"3D Object Setup/Bones Mode/4": "反本地",
|
||||
"3D Object Setup/Bones Mode/5": "反世界",
|
||||
"3D Object Setup/Bones Mode/6": "反世界视图",
|
||||
"3D Object Setup/Bones Mode/7": "反世界视图投影",
|
||||
"2D Sprite Setup/Popup Menu/6": "保存槽..."
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
{
|
||||
"File IO/Dialogs/24": "The Virtools file already exists, \\ndo you want to overwrite it ?",
|
||||
"Schematic/Dialog/15": "Confirm Scene(s) Deletion",
|
||||
"Schematic/Dialog/22": "Saving Behavioral Graphs"
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
{
|
||||
"File IO/Dialogs/24": "Virtools 文件已经存在,\\n您要覆盖它吗?",
|
||||
"Schematic/Dialog/15": "确认删除场景",
|
||||
"Schematic/Dialog/22": "保存行为图"
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
{
|
||||
"CK/Class Names/44": "3D Point Cloud",
|
||||
"CK/Class Names 2/44": "3D Point Clouds",
|
||||
"Common/Dialogs/12": "Move to separated window\\t(Ctrl+Drag)",
|
||||
"Schematic/Toolbar/5": "Show Active",
|
||||
"Schematic/Toolbar/6": "Show Colored",
|
||||
"Schematic/Popup/5": "Set Color\\tCtrl+R",
|
||||
"Schematic/Popup/6": "Remove Color\\tCtrl+Shift+R",
|
||||
"Schematic/Popup/7": "Repair",
|
||||
"Schematic/Popup/8": "Sort",
|
||||
"Schematic/Popup/9": "Sort By Name\\tCtrl+1",
|
||||
"Schematic/Popup/10": "Sort By Owner\\tCtrl+2",
|
||||
"Schematic/Popup/11": "Sort By Color\\tCtrl+3",
|
||||
"Schematic/Popup/12": "Sort By Class ID\\tCtrl+4",
|
||||
"Schematic/Popup/13": "Sort By Activity\\tCtrl+5",
|
||||
"Schematic/Popup/14": "Sort By Priority\\tCtrl+6",
|
||||
"Schematic/Popup/15": "Invert Order\\tCtrl+7",
|
||||
"Schematic/Popup/65": "Alternate Paste\\tCtrl+Alt+V",
|
||||
"Schematic/Popup/76": "Update all Scripts from File\\tCtrl+U",
|
||||
"Schematic/Popup/77": "Update Script(s) from File\\tShift+U",
|
||||
"Schematic/Popup/78": "Add Mark\\tCtrl+F2",
|
||||
"Schematic/Popup/79": "Activate Script(s)",
|
||||
"Schematic/Popup/80": "Deactivate Script(s)",
|
||||
"Schematic/Popup/81": "Reset Script(s)",
|
||||
"Schematic/Popup/82": "\\t(double-click on object icon)",
|
||||
"Schematic/Tooltips/8": "Show All Scripts (Alt+1)",
|
||||
"Schematic/Tooltips/9": "Hide All Scripts (Alt+2)",
|
||||
"Schematic/Tooltips/10": "Hide Selected Scripts (Alt+3)",
|
||||
"Schematic/Tooltips/11": "Hide Unselected Scripts (Alt+4)",
|
||||
"Schematic/Tooltips/12": "Show Active Scripts Only (Alt+5)",
|
||||
"Schematic/Tooltips/13": "Hide scripts which colors is not the same as those of selected scripts (Alt+6)",
|
||||
"Preferences/Schematic/29": "Building Block to Replace Color",
|
||||
"Preferences/Schematic/30": "Trace Color",
|
||||
"Preferences/Schematic/31": "Easy Finding Parameter Operation Dialog",
|
||||
"Preferences/Schematic/32": "Quick Finding Parameter Operation Dialog",
|
||||
"Preferences/Schematic/33": "Path Finding Link Mode",
|
||||
"Preferences/Schematic/34": "Save IOs Positions",
|
||||
"Preferences/Schematic/35": "Save Parameters Positions",
|
||||
"Preferences/Schematic/36": "Main",
|
||||
"Preferences/Schematic/37": "Colors",
|
||||
"Preferences/Schematic/38": "Automatic Scroll Delay (ms)",
|
||||
"Preferences/Schematic/39": "Enable Tooltips. Delay(ms):",
|
||||
"Preferences/Schematic/40": "Rollover List on Group",
|
||||
"Setups/17": "Show In Level Manager",
|
||||
"Setups/18": "Show In Hierarchy Manager",
|
||||
"Setups/19": "Show in 3D Layout",
|
||||
"Setups/20": "On Selection",
|
||||
"Setups/21": "On Hierarchy",
|
||||
"Material Setup/Buttons/5": "Both Sided Lighting",
|
||||
"Material Setup/Buttons/6": "(OpenGL Only)",
|
||||
"Mesh Setup/Blend Shapes/0": "Blend shapes weights",
|
||||
"Mesh Setup/Blend Shapes/1": "Index",
|
||||
"Mesh Setup/Blend Shapes/2": "Name",
|
||||
"Mesh Setup/Blend Shapes/3": "Weight",
|
||||
"Level View/Popup Menu/30": "Remove",
|
||||
"Level View/Popup Menu/31": "Show Setup"
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
{
|
||||
"CK/Class Names/44": "3D点云",
|
||||
"CK/Class Names 2/44": "3D点云",
|
||||
"Common/Dialogs/12": "移到分离窗口\\t(Ctrl+拖动)",
|
||||
"Schematic/Toolbar/5": "显示活动",
|
||||
"Schematic/Toolbar/6": "显示彩色",
|
||||
"Schematic/Popup/5": "设置颜色\\tCtrl+R",
|
||||
"Schematic/Popup/6": "移除颜色\\tCtrl+Shift+R",
|
||||
"Schematic/Popup/7": "修复",
|
||||
"Schematic/Popup/8": "排序",
|
||||
"Schematic/Popup/9": "按名称排序\\tCtrl+1",
|
||||
"Schematic/Popup/10": "按所有者排序\\tCtrl+2",
|
||||
"Schematic/Popup/11": "按颜色排序\\tCtrl+3",
|
||||
"Schematic/Popup/12": "按类ID排序\\tCtrl+4",
|
||||
"Schematic/Popup/13": "按活动排序\\tCtrl+5",
|
||||
"Schematic/Popup/14": "按优先级排序\\tCtrl+6",
|
||||
"Schematic/Popup/15": "反转顺序\\tCtrl+7",
|
||||
"Schematic/Popup/65": "备选粘贴\\tCtrl+Alt+V",
|
||||
"Schematic/Popup/76": "从文件更新所有脚本\\tCtrl+U",
|
||||
"Schematic/Popup/77": "从文件更新脚本\\tShift+U",
|
||||
"Schematic/Popup/78": "添加标记\\tCtrl+F2",
|
||||
"Schematic/Popup/79": "激活脚本",
|
||||
"Schematic/Popup/80": "停用脚本",
|
||||
"Schematic/Popup/81": "重置脚本",
|
||||
"Schematic/Popup/82": "\\t(在对象图标上双击)",
|
||||
"Schematic/Tooltips/8": "显示所有脚本(Alt+1)",
|
||||
"Schematic/Tooltips/9": "隐藏所有脚本(Alt+2)",
|
||||
"Schematic/Tooltips/10": "隐藏所选脚本(Alt+3)",
|
||||
"Schematic/Tooltips/11": "隐藏未选中脚本(Alt+4)",
|
||||
"Schematic/Tooltips/12": "只显示活动脚本(Alt+5)",
|
||||
"Schematic/Tooltips/13": "隐藏颜色与所选脚本不同的脚本(Alt+6)",
|
||||
"Preferences/Schematic/29": "替换颜色的建筑模块",
|
||||
"Preferences/Schematic/30": "追踪颜色",
|
||||
"Preferences/Schematic/31": "参数操作对话框易查找",
|
||||
"Preferences/Schematic/32": "参数操作对话框快速查找",
|
||||
"Preferences/Schematic/33": "路径查找链接模式",
|
||||
"Preferences/Schematic/34": "保存IO位置",
|
||||
"Preferences/Schematic/35": "保存参数位置",
|
||||
"Preferences/Schematic/36": "主页",
|
||||
"Preferences/Schematic/37": "颜色",
|
||||
"Preferences/Schematic/38": "自动滚动延迟(毫秒)",
|
||||
"Preferences/Schematic/39": "启用工具提示。延迟(毫秒):",
|
||||
"Preferences/Schematic/40": "组上鼠标悬停列表",
|
||||
"Setups/17": "在层管理器中显示",
|
||||
"Setups/18": "在层次结构管理器中显示",
|
||||
"Setups/19": "在3D布局中显示",
|
||||
"Setups/20": "在选择时",
|
||||
"Setups/21": "在层次结构上",
|
||||
"Material Setup/Buttons/5": "双面照明",
|
||||
"Material Setup/Buttons/6": "(仅OpenGL)",
|
||||
"Mesh Setup/Blend Shapes/0": "混合形状权重",
|
||||
"Mesh Setup/Blend Shapes/1": "索引",
|
||||
"Mesh Setup/Blend Shapes/2": "名称",
|
||||
"Mesh Setup/Blend Shapes/3": "权重",
|
||||
"Level View/Popup Menu/30": "删除",
|
||||
"Level View/Popup Menu/31": "显示设置"
|
||||
}
|
||||
51
README.md
51
README.md
@ -1,52 +1,3 @@
|
||||
# Virtools Translation
|
||||
|
||||
Provide I18N feature for an abandoned ancient game engine - Virtools.
|
||||
|
||||
According to Virtools directory hierarchy, it should has I18N feature. However, all Virtools distribution, including pirate and official ones, only provide `English.nlp` as the only I18N file in default. The aim of this project is that let Virtools support more languages by providing more I18N files, e.g. NLP files.
|
||||
|
||||
The nature of NLP file is just a Zlib compressed text file with some extra encryptions. After doing decompression and decryption, what we got is a plain text file with specific format. I believe this syntax is invented by Virtools self, because it use a bunch of code to parse this text file according to the decompile result. However, for me, now, I can use various methods to do these things respectively. Each methods is the best solution in its scope. That's the reason why this repository is consisted by a bunch of projects.
|
||||
|
||||
## Hierarchy
|
||||
|
||||
This repository consist of these parts:
|
||||
|
||||
* `NlpCodec`: The codec for the first step of NLP file processing. It can do decompression and decryption stuff for NLP, also including the reversed operation, decompression and encryption.
|
||||
* `NlpParser`: A Java written ANTLR parser which can recognised the syntax of plain text file output by `NlpCodec`, and convert it to a neasted JSON format for following processing. Please note we only use this project for vanilla NLP file parsing. New generated NLP files are built by an ordinary Python script because we do not need to process too much special scenarios when writing NLP files.
|
||||
* `NlpProc`: A collection of Python scripts served for this repository.
|
||||
- Convert the JSON between the nested JSON output by NlpParser and un-nested JSON (flat JSON / plain JSON) to let it more acceptable for the most of common I18N software.
|
||||
- Output NLP text file when compiling translation.
|
||||
|
||||
## How to Translate
|
||||
|
||||
I take `zh-cn` (Chinese) as a example. Navigate to NlpTr folder first. and you will find following files.
|
||||
|
||||
* `VT25.zh-cn.json`
|
||||
* `VT30.zh-cn.json`
|
||||
* `VT35.zh-cn.json`
|
||||
* `VT40.zh-cn.json`
|
||||
* `VT50.zh-cn.json`
|
||||
|
||||
The only things you need to do is translate these JSON files.
|
||||
|
||||
## How to Add Language
|
||||
|
||||
Contact the owner of repository, or follow the manual `NlpTr/README.md` when owner went off.
|
||||
|
||||
## How the Files Generated in NlpTr
|
||||
|
||||
This section is not suit for beginner.
|
||||
|
||||
0. Run `./Scripts/compile_codec.sh` to compile NlpCodec
|
||||
0. Run `./Scripts/compile_parser.sh` to compile NlpParser
|
||||
0. Run `./Scripts/generate_source.sh` to generate the files located in NlpTr.
|
||||
|
||||
## How We Generate NLP Files when Publishing
|
||||
|
||||
This section is not suit for beginner.
|
||||
|
||||
0. Run `./Scripts/compile_codec.sh` to compile NlpCodec. Skip if you have compiled.
|
||||
0. Run `./Scripts/compile_tr.sh`
|
||||
|
||||
## Can I Use This on Windows
|
||||
|
||||
Use MSYS2.
|
||||
This is a part of plan...
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
cd NlpCodec
|
||||
cd NlpEncoder
|
||||
mkdir out
|
||||
cd out
|
||||
cmake ..
|
||||
make
|
||||
cd ../..
|
||||
@ -1,4 +1,3 @@
|
||||
cd NlpParser
|
||||
antlr4 Nlp.g4
|
||||
javac Nlp*.java
|
||||
cd ..
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
cd NlpTr
|
||||
mkdir out
|
||||
cd ../NlpProc
|
||||
python3 NlpJsonEncoder.py
|
||||
cd ..
|
||||
|
||||
cd NlpTr/out
|
||||
for file in *.txt
|
||||
do
|
||||
if test -f $file
|
||||
then
|
||||
txt_file=$file
|
||||
nlp_file=$(basename $file .txt)".nlp"
|
||||
../../NlpCodec/out/NlpCodec encode $txt_file $nlp_file
|
||||
fi
|
||||
done
|
||||
cd ../..
|
||||
@ -1,18 +0,0 @@
|
||||
if [ $# -ne 1 ]
|
||||
then
|
||||
echo "[ERR] invalid arguments"
|
||||
echo "Syntax"
|
||||
echo ""
|
||||
echo "./create_new_tr.sh <lang-symbol>"
|
||||
echo "<lang-symbol>: your preferred language symbol. such as en, de, zh-cn..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd NlpTr
|
||||
cp VT25.template.json "VT25."$1".json"
|
||||
cp VT30.template.json "VT30."$1".json"
|
||||
cp VT35.template.json "VT35."$1".json"
|
||||
cp VT40.template.json "VT40."$1".json"
|
||||
cp VT50.template.json "VT50."$1".json"
|
||||
cd ..
|
||||
echo "DONE"
|
||||
@ -1,17 +1,11 @@
|
||||
./NlpCodec/out/NlpCodec decode NlpSrc/VT25.nlp NlpSrc/VT25.txt
|
||||
./NlpCodec/out/NlpCodec decode NlpSrc/VT30.nlp NlpSrc/VT30.txt
|
||||
./NlpCodec/out/NlpCodec decode NlpSrc/VT35.nlp NlpSrc/VT35.txt
|
||||
./NlpCodec/out/NlpCodec decode NlpSrc/VT40.nlp NlpSrc/VT40.txt
|
||||
./NlpCodec/out/NlpCodec decode NlpSrc/VT50.nlp NlpSrc/VT50.txt
|
||||
./NlpEncoder/out/NlpEncoder uncompress NlpSrc/VT25.nlp NlpSrc/VT25.txt
|
||||
./NlpEncoder/out/NlpEncoder uncompress NlpSrc/VT35.nlp NlpSrc/VT35.txt
|
||||
./NlpEncoder/out/NlpEncoder uncompress NlpSrc/VT40.nlp NlpSrc/VT40.txt
|
||||
./NlpEncoder/out/NlpEncoder uncompress NlpSrc/VT50.nlp NlpSrc/VT50.txt
|
||||
|
||||
cd NlpParser
|
||||
java MainRunner ../NlpSrc/VT25.txt ../NlpSrc/VT25.json
|
||||
java MainRunner ../NlpSrc/VT30.txt ../NlpSrc/VT30.json
|
||||
java MainRunner ../NlpSrc/VT35.txt ../NlpSrc/VT35.json
|
||||
java MainRunner ../NlpSrc/VT40.txt ../NlpSrc/VT40.json
|
||||
java MainRunner ../NlpSrc/VT50.txt ../NlpSrc/VT50.json
|
||||
cd ..
|
||||
|
||||
cd NlpProc
|
||||
python3 NlpJsonDecoder.py
|
||||
java NlpRunner ../NlpSrc/VT25.txt ../NlpSrc/VT25.json
|
||||
java NlpRunner ../NlpSrc/VT35.txt ../NlpSrc/VT35.json
|
||||
java NlpRunner ../NlpSrc/VT40.txt ../NlpSrc/VT40.json
|
||||
java NlpRunner ../NlpSrc/VT50.txt ../NlpSrc/VT50.json
|
||||
cd ..
|
||||
Loading…
x
Reference in New Issue
Block a user