Skip to content
Commits on Source (2)
......@@ -11,8 +11,10 @@
module Mbackup
open System
open System.IO
open System.Diagnostics
open System.Text.RegularExpressions;
open System.Diagnostics.CodeAnalysis
open Argu
......@@ -20,12 +22,14 @@ let ExitBadParam = 1
let ExitTimeout = 2
let ExitIOError = 3
[<SuppressMessage("*", "UnionCasesNames")>]
type CLIArguments =
| [<AltCommandLine("-n")>] Dry_Run
| Target of backupTarget: string
| Remote_User of remoteUser: string
| [<AltCommandLine("-i")>] Itemize_Changes
| Node_Name of nodeName: string
| Ssh_Key of sshKeyFilename: string
with
interface IArgParserTemplate with
member s.Usage =
......@@ -34,6 +38,8 @@ with
| Target _ -> "rsync target, could be local dir or remote ssh dir"
| Remote_User _ -> "remote linux user to own the backup files"
| Itemize_Changes _ -> "add -i option to rsync"
| Node_Name _ -> "local node's name, used in remote logging"
| Ssh_Key _ -> "ssh private key, used when backup to remote ssh node"
type Logger() =
let mutable level = Logger.DEBUG
......@@ -96,24 +102,96 @@ let ToMingwPath (windowsPath: string) =
result
let EnsureDir (path: string) = if path.EndsWith "/" then path else path + "/"
let EnsureWinDir (path: string) = if path.EndsWith "\\" then path else path + "\\"
let appDataRoamingDir = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) |> ToMingwPath |> EnsureDir
let programDataDir = GetEnv "PROGRAMDATA" |> ToMingwPath |> EnsureDir
let appDataLocalDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) |> ToMingwPath |> EnsureDir
let programDataDirWin = GetEnv "PROGRAMDATA" |> EnsureWinDir
let programDataDir = ToMingwPath programDataDirWin
let appDataLocalDirWin = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) |> EnsureWinDir
let appDataLocalDir = appDataLocalDirWin |> ToMingwPath
let mbackupInstallDirWin = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles) |> EnsureDir |> fun s -> s + "mbackup"
let mbackupInstallDir = mbackupInstallDirWin |> ToMingwPath
let userHomeWin =
GetEnvDefault "HOME" (Environment.GetFolderPath(Environment.SpecialFolder.UserProfile))
|> EnsureWinDir
let userHome = userHomeWin |> ToMingwPath
//let userConfigDir = appDataRoamingDir + "mbackup/"
let userConfigDirWin = programDataDirWin + "mbackup\\"
let userConfigDir = programDataDir + "mbackup/"
let runtimeDirWin = appDataLocalDirWin + "mbackup\\"
let runtimeDir = appDataLocalDir + "mbackup/"
let isLocalTarget (target: string) = target.StartsWith "/"
// expand user file to mingw64 rsync supported path.
// abc -> /cygdrive/c/Users/<user>/abc
// ^Documents -> expand to Documents path.
// ^Downloads -> expand to Downloads path.
// etc
let expandUserFile (fn: string) =
let fn =
let documentsDir = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) |> ToMingwPath |> EnsureDir
let picturesDir = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures) |> ToMingwPath |> EnsureDir
let desktopDir = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory) |> ToMingwPath |> EnsureDir
let fn = Regex.Replace(fn, "^My Documents/", documentsDir, RegexOptions.IgnoreCase)
let fn = Regex.Replace(fn, "^Documents/", documentsDir, RegexOptions.IgnoreCase)
let fn = Regex.Replace(fn, "^我的文档/", documentsDir)
let fn = Regex.Replace(fn, "^文档/", documentsDir)
let fn = Regex.Replace(fn, "^My Pictures/", picturesDir, RegexOptions.IgnoreCase)
let fn = Regex.Replace(fn, "^Pictures/", picturesDir, RegexOptions.IgnoreCase)
let fn = Regex.Replace(fn, "^图片/", picturesDir)
let fn = Regex.Replace(fn, "^Desktop/", desktopDir, RegexOptions.IgnoreCase)
let fn = Regex.Replace(fn, "^桌面/", desktopDir)
fn
if fn.StartsWith("/") then
fn
else
userHome + fn
// generate mbackup.list file
let generateMbackupList =
// TODO run make to update mbackup.list. only update if source file have changed.
null
let generateMbackupList (logger: Logger) =
// TODO how to only regenerate if source file have changed? should I bundle GNU make with mbackup?
// just compare mbackup.list mtime with its source files?
let mbackupDefaultList = userConfigDirWin + "mbackup-default.list"
let mbackupLocalList = userConfigDirWin + "local.list"
let mbackupUserDefaultList = userConfigDirWin + "user-default.list"
let mbackupList = runtimeDirWin + "mbackup.list"
// local functions
let dropEmptyLinesAndComments = Seq.filter (fun (line: string) -> not (line.TrimStart().StartsWith("#") || line.TrimEnd().Equals("")))
let readMbackupListFile fn = File.ReadAllLines(fn) |> dropEmptyLinesAndComments
try
let defaultListLines = readMbackupListFile mbackupDefaultList |> Seq.map ToMingwPath
let localListLinesMaybe =
try
let lines = readMbackupListFile mbackupLocalList |> Seq.map ToMingwPath
(true, lines)
with
| :? System.IO.FileNotFoundException ->
(true, Seq.empty)
| ex ->
logger.Error "Read mbackupLocalList failed: %s" ex.Message
(false, Seq.empty)
match localListLinesMaybe with
| (false, _) -> failwith "Read mbackup local.list file failed"
| (true, localListLines) ->
let userDefaultListLines = readMbackupListFile mbackupUserDefaultList |> Seq.map expandUserFile
let allLines = Seq.append (Seq.append defaultListLines localListLines) userDefaultListLines
// For mbackup-default.list and local.list, exclude empty lines and comment lines.
// skip and give a warning on non-absolute path.
// For user-default.list, auto prefix user's home dir, auto expand Documents, Downloads etc special folder.
File.WriteAllLines(mbackupList, allLines)
logger.Info "%s written" mbackupList
true
with
| ex ->
logger.Error "Read/write mbackup list file failed: %s" ex.Message
false
// append string s to list if pred is true
let appendWhen (pred: bool) (lst: string list) (s: string) = if pred then List.append lst [s] else lst
......@@ -128,8 +206,8 @@ let main argv =
let logger = Logger()
logger.Info "userConfigDir=%s" userConfigDir
logger.Info "runtimeDir=%s" runtimeDir
logger.Info "using user config dir: %s" userConfigDirWin
logger.Info "using runtime dir: %s" runtimeDirWin
let rsyncCmd: string list = []
let rsyncCmd = appendWhen dryRun rsyncCmd "--dry-run"
......@@ -137,7 +215,8 @@ let main argv =
let rsyncCmd = List.append rsyncCmd ("-h --stats -togr --delete --delete-excluded --ignore-missing-args".Split [|' '|] |> Array.toList)
let mbackupFile = runtimeDir + "mbackup.list"
generateMbackupList |> ignore
if not (generateMbackupList logger) then
failwith "Generate mbackup.list failed"
let rsyncCmd = List.append rsyncCmd [sprintf "--files-from=%s" mbackupFile]
let excludeFile = userConfigDir + "mbackup-default.exclude"
......@@ -152,27 +231,21 @@ let main argv =
let mbackupInstallDirWinTest = "D:\\downloads\\apps\\mbackupTest\\"
let mbackupInstallDirTest = mbackupInstallDirWinTest |> ToMingwPath |> EnsureDir
let sshExeFile = mbackupInstallDirTest + "rsync-w64/usr/bin/ssh.exe"
let sshConfigFile = userConfigDir + "ssh_config"
let sshPrivateKeyFile = userConfigDir + "ssh_id_rsa"
let sshConfigFile = userHome + ".ssh/config"
let sshPrivateKeyFile = results.GetResult(Ssh_Key, defaultValue = userHome + ".ssh/id_rsa") |> ToMingwPath
let rsyncCmd = List.append rsyncCmd [sprintf "-e \"%s -F %s -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\"" sshExeFile sshConfigFile sshPrivateKeyFile]
let backupTarget = results.GetResult (Target, defaultValue = Environment.GetEnvironmentVariable "TARGET")
match backupTarget with
| null ->
logger.Error "TARGET is not defined"
logger.Error "TARGET is not defined%s" ""
ExitBadParam
| _ ->
let backupTarget = ToMingwPath backupTarget
let rsyncCmd =
if not (isLocalTarget backupTarget)
then
let nodeName =
match results.GetResult Node_Name with
| null ->
match GetEnv "NODE_NAME" with
| null -> Net.Dns.GetHostName()
| X -> X
| X -> X
let nodeName = results.GetResult(Node_Name, defaultValue = Net.Dns.GetHostName())
let remoteLogFile = sprintf "/var/log/mbackup/%s.log" nodeName
let remoteUser = results.GetResult (Remote_User, defaultValue = Environment.UserName)
let rsyncCmd = List.append rsyncCmd [sprintf "--remote-option=--log-file=%s" remoteLogFile]
......@@ -196,9 +269,9 @@ let main argv =
if proc.WaitForExit Int32.MaxValue then
proc.ExitCode
else
logger.Error "mbackup timed out while waiting for rsync to complete"
logger.Error "mbackup timed out while waiting for rsync to complete%s" ""
ExitTimeout
with
| _ ->
logger.Error "Create runtime dir failed"
logger.Error "Create runtime dir failed%s" ""
ExitIOError
# dirs to backup
/cygdrive/c/ProgramData/mbackup
/cygdrive/c/path/to/dir
/cygdrive/d/path/to/dir
C:\ProgramData\mbackup
C:\path\to\dir
D:\path\to\dir
......@@ -5,6 +5,11 @@
#############################################
Pictures/Saved Pictures/
Documents/
Desktop/
#downloads dir is not supported yet. it will just be prefixed by user's profile dir.
#if user moved this dir to other location, they should add dir to local.list.
Downloads/
#############################################
# below are file list from linux environment
......
......@@ -6,6 +6,16 @@ Time-stamp: <2019-11-12>
** 2019-11-13 how to run it in dev env?
dotnet run -- --dry-run --itemize-changes --target d:\backup
dotnet run -- -n -i --target d:\backup
dotnet run -- -n --target d:\backup
try an ssh run:
dotnet run -- -n --target root@sylecn01.emacsos.com:/data/backup/PC-backup/B75I3/
it works.
file list works.
ssh transfer works.
remote logging works.
** 2019-11-12 docs
- rsync
......@@ -44,9 +54,12 @@ dotnet run -- -n -i --target d:\backup
MyPictures
DesktopDirectory
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)
-
- FSharpLint
http://fsprojects.github.io/FSharpLint/
http://fsprojects.github.io/FSharpLint/index.html
* later :entry:
** 2019-11-14 supports expand Downloads dir in user-default.list
* current :entry:
**
** 2019-11-13 extra user default list.
......@@ -209,4 +222,5 @@ C:\ProgramData\mbackup\user-default.list
it contains hostname.
-
* done :entry:
** 2019-11-13 build mbackup.list file from file list and exclude lists.
* wontfix :entry: