Commit d8cf9beb authored by Yuanle Song's avatar Yuanle Song
Browse files

WIP ported mbackup python code to F# code

still missing many pieces and missing windows specific handling.
parent d179f801
obj/
bin/
.ionide/
// Learn more about F# at http://fsharp.org
//
// - backup file list
// /%appdata%/mbackup/mbackup-default.list
// /%appdata%/mbackup/user-default.list
// /%appdata%/mbackup/local.list (optional)
// - exclude pattern
// /%appdata%/mbackup/mbackup-default.exclude
// /%appdata%/mbackup/local.exclude (optional)
open System
open System.Diagnostics
open Argu
let ExitBadParam = 1
let ExitTimeout = 2
type CLIArguments =
| DryRun
| Cron
| Target of backupTarget: string
with
interface IArgParserTemplate with
member s.Usage =
match s with
| DryRun _ -> "only show what will be done, do not transfer any file"
| Cron _ -> "run in cron mode, do not ask user any questions"
| Target _ -> "rsync target, could be local dir or remote ssh dir"
type Logger() =
let mutable level = Logger.DEBUG
static member DEBUG = 10
static member INFO = 20
static member WARNING = 30
static member ERROR = 40
static member LevelToString level =
match level with
| 10 -> "DEBUG"
| 20 -> "INFO"
| 30 -> "WARNING"
| 40 -> "ERROR"
| _ -> failwith (sprintf "Unknown log level: %d" level)
member this.Level = level
member this.LogMaybe level fmt =
if this.Level <= level then
Printf.ksprintf (
fun s ->
let time = DateTime.Now
printfn "%02d:%02d:%02d %s %s" time.Hour time.Minute time.Second (Logger.LevelToString level) s)
fmt
else
Printf.ksprintf (fun _ -> printfn "") fmt
member this.SetLevel level =
this.Level = level
member this.Debug = this.LogMaybe Logger.DEBUG
member this.Info = this.LogMaybe Logger.INFO
member this.Warning = this.LogMaybe Logger.WARNING
member this.Error = this.LogMaybe Logger.ERROR
let GetEnv (varName: string) = Environment.GetEnvironmentVariable varName
let GetEnvDefault (varName: string) (defaultValue: string) =
let value = Environment.GetEnvironmentVariable varName
match value with
| null -> defaultValue
| "" -> defaultValue
| _ -> value
// Convert windows path to Mingw64 path
let ToMingwPath windowsPath =
// TODO implement me
windowsPath
let appDataRoamingDir = ToMingwPath (Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData))
let appDataLocalDir = ToMingwPath (Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData))
// TODO make sure dir ends with /
let userConfigDir = appDataRoamingDir + "/mbackup/"
let runtimeDir = appDataLocalDir
let isLocalTarget (target: string) = target.StartsWith "/"
[<EntryPoint>]
let main argv =
let errorHandler = ProcessExiter(colorizer = function ErrorCode.HelpText -> None | _ -> Some ConsoleColor.Red)
let parser = ArgumentParser.Create<CLIArguments>(programName = "mbackup.exe", errorHandler = errorHandler)
let results = parser.Parse argv
let cron = results.Contains Cron
let dryRun = results.Contains DryRun
let logger = Logger()
logger.Info "userConfigDir=%s" userConfigDir
logger.Info "runtimeDir=%s" runtimeDir
let rsyncCmd: string list = []
let rsyncCmd = if dryRun then List.append rsyncCmd ["--dry-run"] else rsyncCmd
let rsyncCmd = List.append rsyncCmd ("-h --stats -air --delete --delete-excluded --ignore-missing-args".Split [|' '|] |> Array.toList)
let mbackupFile = runtimeDir + "mbackup.list"
let rsyncCmd = List.append rsyncCmd [sprintf "--files-from=%s" mbackupFile]
let excludeFile = userConfigDir + "mbackup-default.exclude"
let rsyncCmd = List.append rsyncCmd [sprintf "--exclude-from=%s" excludeFile]
let localExcludeFile = userConfigDir + "local.list"
let rsyncCmd = if IO.File.Exists localExcludeFile then List.append rsyncCmd [sprintf "--exclude-from=%s" localExcludeFile] else rsyncCmd
let localLogFile = runtimeDir + "mbackup.log"
let rsyncCmd = List.append rsyncCmd [sprintf "--log-file=%s" localLogFile]
let backupTarget = results.GetResult (Target, defaultValue = Environment.GetEnvironmentVariable "TARGET")
match backupTarget with
| null ->
logger.Error "TARGET is not defined"
ExitBadParam
| _ ->
let backupTarget = ToMingwPath backupTarget
let rsyncCmd =
if not (isLocalTarget backupTarget)
then
let hostname: string = Net.Dns.GetHostName()
let nodeName: string = GetEnvDefault "NODE_NAME" hostname
let remoteLogFile = sprintf "/var/log/mbackup/%s.log" nodeName
List.append rsyncCmd [sprintf "--remote-option=--log-file=%s" remoteLogFile]
else
rsyncCmd
let rsyncCmd = List.append rsyncCmd ["/"]
let rsyncCmd = List.append rsyncCmd [backupTarget]
let rsyncArgs = rsyncCmd |> String.concat " "
// TODO print rsyncArgs in the same Info call when Info supports multiple params.
logger.Info "Note: if you run the following rsync command yourself, make sure the generated file list (%s) is up-to-date." mbackupFile
let rsyncExe = "D:\\downloads\\apps\\rsync-w64-3.1.3-2-standalone\\usr\\bin\\rsync.exe"
let echoExe = "C:\\Program Files\\Git\\usr\\bin\\echo.exe"
let proc = Process.Start(echoExe, rsyncArgs)
logger.Info "%s" (rsyncExe + rsyncArgs)
if proc.WaitForExit Int32.MaxValue then
proc.ExitCode
else
logger.Error "mbackup timed out while waiting for rsync to complete"
ExitTimeout
*.exe
*.msi
*.zip
*.rar
*.7z
*.tar.gz
*.iso
*.img
*.vmdk
*.vdi
*.vhd
*.sav
.cabal-sandbox/
.stack-work/
bower_components/
node_modules/
*.py[co]
__pycache__
.tox/
.venv/
.venv-pypy/
.pytest_cache/
lib/python2.*/
lib/python3.*/
bin/python*
utils/virtualenv-*/
django/
local/
doc/_build/
*.class
build/
dist/
target/
#target/uberjar/
#target/classes/
.gradle/
_build/
_rel/
deps/
bin/
obj/
#testing-only/
*.elc
*.egg-info
*.[oa]
*.hi
TAGS
a.out
*.fasl
*~
*#
*.swp
tmp
temp
tmp.*
temp.*
t[0-9]
t[0-9].*
.coverage
.sconsign.dblite
CMakeFiles/
CTestTestfile.cmake
CMakeCache.txt
cmake_install.cmake
.java-project.el
__history/
nohup.out
octave-core
core
vgcore.*
.sass-cache/
image-dired/
# dirs to backup
/cygdrive/c/path/to/dir
/cygdrive/d/path/to/dir
# user dir prefix will be /cygdrive/%userprofile%/, usually /cygdrive/c/Users/<user>/
#############################################
# windows specific files
#############################################
Pictures/Saved Pictures/
#############################################
# below are file list from linux environment
#############################################
# shell
.profile
.bashrc
.bash.d/
.inputrc
.screenrc
.zshrc
# passwords, keys
.netrc
.ssh/
.gnupg/
.aws/
.azure/
.pgpass
.pypirc
.acme.sh/
.docker/config.json
.kube/config
.erlang.cookie
.lein/credentials.clj.gpg
# editor
.vimrc
.vim/
.emacs
.emacs.el
.emacs.d/
.gnus.el
.aspell.en.prepl
.aspell.en.pws
# mail
.muttrc
.forward
.fetchmailrc
.mailcap
.mime.types
# developer
.gitconfig
.gitignore
.hgrc
.pip/pip.conf
.pylintrc
.npmrc
.gemrc
.m2/settings.xml
.gradle/gradle.properties
.stack/stack.yaml
.stack/config.yaml
.cabal/config
.sbclrc
.roswell/init.lisp
.lein/profiles.clj
# Xorg and WM
.xinitrc
.Xresources
.xsessionrc
.xmodmaprc
.fonts.conf
.fonts
.i3/
.config/i3status/
.config/openbox/
.config/fbpanel/
.gtk-bookmarks
.gtkrc-2.0
.xbindkeysrc
.xbindkeysrc.scm
.xmobarrc
.xmonad/xmonad.hs
.cache/ibus/pinyin/user-1.0.db
# backup dirs
bak/
texts/
StudioProjects/
# misc, other
.config/dconf/user
.aptitude/config
.lftp/bookmarks
.wgetrc
.curlrc
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.0</TargetFramework>
<RootNamespace>mbackup_for_windows</RootNamespace>
</PropertyGroup>
<ItemGroup>
<Compile Include="Program.fs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Argu" Version="5.5.0" />
</ItemGroup>
</Project>
...@@ -3,9 +3,99 @@ ...@@ -3,9 +3,99 @@
Time-stamp: <2019-11-12> Time-stamp: <2019-11-12>
#+STARTUP: content #+STARTUP: content
* notes :entry: * notes :entry:
** 2019-11-12 docs
- rsync
https://www.samba.org/ftp/rsync/rsync.html
- Basic Editing in Visual Studio Code
https://code.visualstudio.com/docs/editor/codebasics
Get Started with F# in Visual Studio Code | Microsoft Docs
https://docs.microsoft.com/en-us/dotnet/fsharp/get-started/get-started-vscode
install Ionide-fsharp extension.
- Argu
http://fsprojects.github.io/Argu/
Tutorial
http://fsprojects.github.io/Argu/tutorial.html
--help doesn't work, need error handler.
Argu/Program.fs at master · fsprojects/Argu · GitHub
https://github.com/fsprojects/Argu/blob/master/samples/Argu.Samples.LS/Program.fs
- F#
F# Language Reference - F# | Microsoft Docs
https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/
F# Collection Types - F# | Microsoft Docs
https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/fsharp-collection-types
fsharp-cheatsheet
https://dungpa.github.io/fsharp-cheatsheet/
Literals - F# | Microsoft Docs
https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/literals
[|' '|] is an array of character, length is 1.
F# list is not the same as array. You can convert array to list via Array.toList
Environment.SpecialFolder
https://docs.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8
ApplicationData
LocalApplicationData
MyDocuments
MyPictures
DesktopDirectory
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)
-
* later :entry: * later :entry:
* current :entry: * current :entry:
** **
** 2019-11-12 make code work in a specific dir. then create an installer.
- install to %programfiles%
- config is saved to %appdata% roaming dir.
- on B75I3, rsync is here.
D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\rsync.exe
scheduled task command is:
<Actions Context="Author">
<Exec>
<Command>D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\rsync.exe</Command>
<Arguments>--stats -togr --chown=sylecn:sylecn --exclude-from=/cygdrive/d/sylecn_docs/texts/configs/rsync-exclude --files-from=/cygdrive/d/sylecn_docs/texts/configs/rsync-file-list --log-file=/cygdrive/d/sylecn_docs/rsync-b75i3.log -e ".\ssh.exe -F c:/users/sylecn/.ssh/config -i c:/Users/sylecn/.ssh/id_rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" / sheni:/data/backup/server-backup/b75i3/</Arguments>
<WorkingDirectory>D:\downloads\apps\rsync-w64-3.1.3-2-standalone\usr\bin\</WorkingDirectory>
</Exec>
</Actions>
- I don't want to bundle python with mbackup for windows, so I will use
C# and dotnet core 3 to create mbackup console executable.
mbackup --cron
# run the command as if run in scheduled task.
mbackup
# run backup command interactively.
create new console project.
dotnet new console -lang=F#
Try F# this time. it's said dotnet have very good F# support.
It's a small program, should be easy to handle.
- implementation
- install-package argu
can't find package.
https://www.nuget.org/packages/Argu
try this:
dotnet add package Argu --version 5.5.0
it works.
- how to change output exe file name?
can't find it. won't fix. default name is mbackup-for-windows.dll/exe
- make rsync command work.
- how to backup only documents in "My Documents" and "Downloads"?
I don't want to backup exe/msi/zip/rar/7z/iso/tar.gz files in those dir.
Only backup pdf/docx/xlsx etc.
Maybe just let user add "My Documents" in local.list.
I won't backup anything by default.
This is not good enough. docs should be backed up by default.
can I backup a zip file if I have Documents/**.zip in exclude list?
- TODO F# doesn't have a free logging framework yet?
logary/LICENSE.md at master · logary/logary · GitHub
https://github.com/logary/logary/blob/master/LICENSE.md
this one is not free software.
-
** 2019-11-12 how to test it? test it in a win 10 VM? ** 2019-11-12 how to test it? test it in a win 10 VM?
- search: lite weight win 10 VM - search: lite weight win 10 VM
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment