From d8cf9beb3d87b1e1cebe9764e52291c1c6534df4 Mon Sep 17 00:00:00 2001 From: Yuanle Song Date: Wed, 13 Nov 2019 01:32:56 +0800 Subject: [PATCH] WIP ported mbackup python code to F# code still missing many pieces and missing windows specific handling. --- .gitignore | 3 + Program.fs | 143 +++++++++++++++++++++++++ mbackup-config/mbackup-default.exclude | 78 ++++++++++++++ mbackup-config/mbackup-default.list | 3 + mbackup-config/user-default.list | 97 +++++++++++++++++ mbackup-for-windows.fsproj | 17 +++ operational | 90 ++++++++++++++++ 7 files changed, 431 insertions(+) create mode 100644 .gitignore create mode 100644 Program.fs create mode 100644 mbackup-config/mbackup-default.exclude create mode 100644 mbackup-config/mbackup-default.list create mode 100644 mbackup-config/user-default.list create mode 100644 mbackup-for-windows.fsproj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff4880b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +obj/ +bin/ +.ionide/ diff --git a/Program.fs b/Program.fs new file mode 100644 index 0000000..0195090 --- /dev/null +++ b/Program.fs @@ -0,0 +1,143 @@ +// Learn more about F# at http://fsharp.org +// +// - backup file list +// /%appdata%/mbackup/mbackup-default.list +// /%appdata%/mbackup/user-default.list +// /%appdata%/mbackup/local.list (optional) +// - exclude pattern +// /%appdata%/mbackup/mbackup-default.exclude +// /%appdata%/mbackup/local.exclude (optional) + +open System +open System.Diagnostics + +open Argu + +let ExitBadParam = 1 +let ExitTimeout = 2 + +type CLIArguments = + | DryRun + | Cron + | Target of backupTarget: string +with + interface IArgParserTemplate with + member s.Usage = + match s with + | DryRun _ -> "only show what will be done, do not transfer any file" + | Cron _ -> "run in cron mode, do not ask user any questions" + | Target _ -> "rsync target, could be local dir or remote ssh dir" + +type Logger() = + let mutable level = Logger.DEBUG + + static member DEBUG = 10 + static member INFO = 20 + static member WARNING = 30 + static member ERROR = 40 + static member LevelToString level = + match level with + | 10 -> "DEBUG" + | 20 -> "INFO" + | 30 -> "WARNING" + | 40 -> "ERROR" + | _ -> failwith (sprintf "Unknown log level: %d" level) + + member this.Level = level + + member this.LogMaybe level fmt = + if this.Level <= level then + Printf.ksprintf ( + fun s -> + let time = DateTime.Now + printfn "%02d:%02d:%02d %s %s" time.Hour time.Minute time.Second (Logger.LevelToString level) s) + fmt + else + Printf.ksprintf (fun _ -> printfn "") fmt + + member this.SetLevel level = + this.Level = level + member this.Debug = this.LogMaybe Logger.DEBUG + member this.Info = this.LogMaybe Logger.INFO + member this.Warning = this.LogMaybe Logger.WARNING + member this.Error = this.LogMaybe Logger.ERROR + +let GetEnv (varName: string) = Environment.GetEnvironmentVariable varName + +let GetEnvDefault (varName: string) (defaultValue: string) = + let value = Environment.GetEnvironmentVariable varName + match value with + | null -> defaultValue + | "" -> defaultValue + | _ -> value + +// Convert windows path to Mingw64 path +let ToMingwPath windowsPath = + // TODO implement me + windowsPath + +let appDataRoamingDir = ToMingwPath (Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)) +let appDataLocalDir = ToMingwPath (Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)) + +// TODO make sure dir ends with / +let userConfigDir = appDataRoamingDir + "/mbackup/" +let runtimeDir = appDataLocalDir + +let isLocalTarget (target: string) = target.StartsWith "/" + +[] +let main argv = + let errorHandler = ProcessExiter(colorizer = function ErrorCode.HelpText -> None | _ -> Some ConsoleColor.Red) + let parser = ArgumentParser.Create(programName = "mbackup.exe", errorHandler = errorHandler) + let results = parser.Parse argv + let cron = results.Contains Cron + let dryRun = results.Contains DryRun + + let logger = Logger() + + logger.Info "userConfigDir=%s" userConfigDir + logger.Info "runtimeDir=%s" runtimeDir + + let rsyncCmd: string list = [] + let rsyncCmd = if dryRun then List.append rsyncCmd ["--dry-run"] else rsyncCmd + let rsyncCmd = List.append rsyncCmd ("-h --stats -air --delete --delete-excluded --ignore-missing-args".Split [|' '|] |> Array.toList) + let mbackupFile = runtimeDir + "mbackup.list" + let rsyncCmd = List.append rsyncCmd [sprintf "--files-from=%s" mbackupFile] + let excludeFile = userConfigDir + "mbackup-default.exclude" + let rsyncCmd = List.append rsyncCmd [sprintf "--exclude-from=%s" excludeFile] + let localExcludeFile = userConfigDir + "local.list" + let rsyncCmd = if IO.File.Exists localExcludeFile then List.append rsyncCmd [sprintf "--exclude-from=%s" localExcludeFile] else rsyncCmd + let localLogFile = runtimeDir + "mbackup.log" + let rsyncCmd = List.append rsyncCmd [sprintf "--log-file=%s" localLogFile] + + let backupTarget = results.GetResult (Target, defaultValue = Environment.GetEnvironmentVariable "TARGET") + match backupTarget with + | null -> + logger.Error "TARGET is not defined" + ExitBadParam + | _ -> + let backupTarget = ToMingwPath backupTarget + let rsyncCmd = + if not (isLocalTarget backupTarget) + then + let hostname: string = Net.Dns.GetHostName() + let nodeName: string = GetEnvDefault "NODE_NAME" hostname + let remoteLogFile = sprintf "/var/log/mbackup/%s.log" nodeName + List.append rsyncCmd [sprintf "--remote-option=--log-file=%s" remoteLogFile] + else + rsyncCmd + + let rsyncCmd = List.append rsyncCmd ["/"] + let rsyncCmd = List.append rsyncCmd [backupTarget] + let rsyncArgs = rsyncCmd |> String.concat " " + // TODO print rsyncArgs in the same Info call when Info supports multiple params. + logger.Info "Note: if you run the following rsync command yourself, make sure the generated file list (%s) is up-to-date." mbackupFile + let rsyncExe = "D:\\downloads\\apps\\rsync-w64-3.1.3-2-standalone\\usr\\bin\\rsync.exe" + let echoExe = "C:\\Program Files\\Git\\usr\\bin\\echo.exe" + let proc = Process.Start(echoExe, rsyncArgs) + logger.Info "%s" (rsyncExe + rsyncArgs) + if proc.WaitForExit Int32.MaxValue then + proc.ExitCode + else + logger.Error "mbackup timed out while waiting for rsync to complete" + ExitTimeout diff --git a/mbackup-config/mbackup-default.exclude b/mbackup-config/mbackup-default.exclude new file mode 100644 index 0000000..74ae6cd --- /dev/null +++ b/mbackup-config/mbackup-default.exclude @@ -0,0 +1,78 @@ +*.exe +*.msi +*.zip +*.rar +*.7z +*.tar.gz +*.iso +*.img +*.vmdk +*.vdi +*.vhd +*.sav + +.cabal-sandbox/ +.stack-work/ + +bower_components/ +node_modules/ + +*.py[co] +__pycache__ +.tox/ +.venv/ +.venv-pypy/ +.pytest_cache/ +lib/python2.*/ +lib/python3.*/ +bin/python* +utils/virtualenv-*/ +django/ +local/ +doc/_build/ + +*.class +build/ +dist/ +target/ +#target/uberjar/ +#target/classes/ +.gradle/ +_build/ +_rel/ +deps/ +bin/ +obj/ + +#testing-only/ + +*.elc +*.egg-info +*.[oa] +*.hi +TAGS +a.out +*.fasl +*~ +*# +*.swp +tmp +temp +tmp.* +temp.* +t[0-9] +t[0-9].* +.coverage +.sconsign.dblite +CMakeFiles/ +CTestTestfile.cmake +CMakeCache.txt +cmake_install.cmake +.java-project.el +__history/ +nohup.out +octave-core +core +vgcore.* +.sass-cache/ +image-dired/ diff --git a/mbackup-config/mbackup-default.list b/mbackup-config/mbackup-default.list new file mode 100644 index 0000000..b8b7e29 --- /dev/null +++ b/mbackup-config/mbackup-default.list @@ -0,0 +1,3 @@ +# dirs to backup +/cygdrive/c/path/to/dir +/cygdrive/d/path/to/dir diff --git a/mbackup-config/user-default.list b/mbackup-config/user-default.list new file mode 100644 index 0000000..41b3230 --- /dev/null +++ b/mbackup-config/user-default.list @@ -0,0 +1,97 @@ +# user dir prefix will be /cygdrive/%userprofile%/, usually /cygdrive/c/Users// + +############################################# +# windows specific files +############################################# +Pictures/Saved Pictures/ + +############################################# +# below are file list from linux environment +############################################# + +# shell +.profile +.bashrc +.bash.d/ +.inputrc +.screenrc +.zshrc + +# passwords, keys +.netrc +.ssh/ +.gnupg/ +.aws/ +.azure/ +.pgpass +.pypirc +.acme.sh/ +.docker/config.json +.kube/config +.erlang.cookie +.lein/credentials.clj.gpg + +# editor +.vimrc +.vim/ +.emacs +.emacs.el +.emacs.d/ +.gnus.el +.aspell.en.prepl +.aspell.en.pws + +# mail +.muttrc +.forward +.fetchmailrc +.mailcap +.mime.types + +# developer +.gitconfig +.gitignore +.hgrc +.pip/pip.conf +.pylintrc +.npmrc +.gemrc +.m2/settings.xml +.gradle/gradle.properties +.stack/stack.yaml +.stack/config.yaml +.cabal/config +.sbclrc +.roswell/init.lisp +.lein/profiles.clj + +# Xorg and WM +.xinitrc +.Xresources +.xsessionrc +.xmodmaprc +.fonts.conf +.fonts +.i3/ +.config/i3status/ +.config/openbox/ +.config/fbpanel/ +.gtk-bookmarks +.gtkrc-2.0 +.xbindkeysrc +.xbindkeysrc.scm +.xmobarrc +.xmonad/xmonad.hs +.cache/ibus/pinyin/user-1.0.db + +# backup dirs +bak/ +texts/ +StudioProjects/ + +# misc, other +.config/dconf/user +.aptitude/config +.lftp/bookmarks +.wgetrc +.curlrc diff --git a/mbackup-for-windows.fsproj b/mbackup-for-windows.fsproj new file mode 100644 index 0000000..247ea76 --- /dev/null +++ b/mbackup-for-windows.fsproj @@ -0,0 +1,17 @@ + + + + Exe + netcoreapp3.0 + mbackup_for_windows + + + + + + + + + + + diff --git a/operational b/operational index d635fb9..1b4e401 100644 --- a/operational +++ b/operational @@ -3,9 +3,99 @@ Time-stamp: <2019-11-12> #+STARTUP: content * notes :entry: +** 2019-11-12 docs +- rsync + https://www.samba.org/ftp/rsync/rsync.html +- Basic Editing in Visual Studio Code + https://code.visualstudio.com/docs/editor/codebasics + + Get Started with F# in Visual Studio Code | Microsoft Docs + https://docs.microsoft.com/en-us/dotnet/fsharp/get-started/get-started-vscode + install Ionide-fsharp extension. +- Argu + http://fsprojects.github.io/Argu/ + Tutorial + http://fsprojects.github.io/Argu/tutorial.html + --help doesn't work, need error handler. + Argu/Program.fs at master · fsprojects/Argu · GitHub + https://github.com/fsprojects/Argu/blob/master/samples/Argu.Samples.LS/Program.fs +- F# + F# Language Reference - F# | Microsoft Docs + https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/ + F# Collection Types - F# | Microsoft Docs + https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/fsharp-collection-types + fsharp-cheatsheet + https://dungpa.github.io/fsharp-cheatsheet/ + Literals - F# | Microsoft Docs + https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/literals + [|' '|] is an array of character, length is 1. + F# list is not the same as array. You can convert array to list via Array.toList + Environment.SpecialFolder + https://docs.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8 + ApplicationData + LocalApplicationData + MyDocuments + MyPictures + DesktopDirectory + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) +- + * later :entry: * current :entry: ** +** 2019-11-12 make code work in a specific dir. then create an installer. +- install to %programfiles% +- config is saved to %appdata% roaming dir. +- on B75I3, rsync is here. + D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\rsync.exe + + scheduled task command is: + + + D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\rsync.exe + --stats -togr --chown=sylecn:sylecn --exclude-from=/cygdrive/d/sylecn_docs/texts/configs/rsync-exclude --files-from=/cygdrive/d/sylecn_docs/texts/configs/rsync-file-list --log-file=/cygdrive/d/sylecn_docs/rsync-b75i3.log -e ".\ssh.exe -F c:/users/sylecn/.ssh/config -i c:/Users/sylecn/.ssh/id_rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" / sheni:/data/backup/server-backup/b75i3/ + D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\ + + +- I don't want to bundle python with mbackup for windows, so I will use + C# and dotnet core 3 to create mbackup console executable. + + mbackup --cron + # run the command as if run in scheduled task. + + mbackup + # run backup command interactively. + + create new console project. + dotnet new console -lang=F# + + Try F# this time. it's said dotnet have very good F# support. + It's a small program, should be easy to handle. +- implementation + - install-package argu + can't find package. + https://www.nuget.org/packages/Argu + try this: + dotnet add package Argu --version 5.5.0 + it works. + - how to change output exe file name? + can't find it. won't fix. default name is mbackup-for-windows.dll/exe + - make rsync command work. + - how to backup only documents in "My Documents" and "Downloads"? + I don't want to backup exe/msi/zip/rar/7z/iso/tar.gz files in those dir. + Only backup pdf/docx/xlsx etc. + + Maybe just let user add "My Documents" in local.list. + I won't backup anything by default. + This is not good enough. docs should be backed up by default. + + can I backup a zip file if I have Documents/**.zip in exclude list? + - TODO F# doesn't have a free logging framework yet? + logary/LICENSE.md at master · logary/logary · GitHub + https://github.com/logary/logary/blob/master/LICENSE.md + this one is not free software. + - + ** 2019-11-12 how to test it? test it in a win 10 VM? - search: lite weight win 10 VM -- GitLab