Program.fs

@@ -11,8 +11,10 @@
 module Mbackup
 
 open System
+open System.IO
 open System.Diagnostics
 open System.Text.RegularExpressions;
+open System.Diagnostics.CodeAnalysis
 
 open Argu
 
@@ -20,12 +22,14 @@ let ExitBadParam = 1
 let ExitTimeout = 2
 let ExitIOError = 3
 
+[<SuppressMessage("*", "UnionCasesNames")>]
 type CLIArguments =
     | [<AltCommandLine("-n")>] Dry_Run
     | Target of backupTarget: string
     | Remote_User of remoteUser: string
     | [<AltCommandLine("-i")>] Itemize_Changes
     | Node_Name of nodeName: string
+    | Ssh_Key of sshKeyFilename: string
 with
     interface IArgParserTemplate with
         member s.Usage =
@@ -34,6 +38,8 @@ with
             | Target _ -> "rsync target, could be local dir or remote ssh dir"
             | Remote_User _ -> "remote linux user to own the backup files"
             | Itemize_Changes _ -> "add -i option to rsync"
+            | Node_Name _ -> "local node's name, used in remote logging"
+            | Ssh_Key _ -> "ssh private key, used when backup to remote ssh node"
 
 type Logger() =
   let mutable level = Logger.DEBUG
@@ -96,24 +102,96 @@ let ToMingwPath (windowsPath: string) =
   result
 
 let EnsureDir (path: string) = if path.EndsWith "/" then path else path + "/"
+let EnsureWinDir (path: string) = if path.EndsWith "\\" then path else path + "\\"
 
 let appDataRoamingDir = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) |> ToMingwPath |> EnsureDir
-let programDataDir = GetEnv "PROGRAMDATA" |> ToMingwPath |> EnsureDir
-let appDataLocalDir =  Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) |> ToMingwPath |> EnsureDir
+let programDataDirWin = GetEnv "PROGRAMDATA" |> EnsureWinDir
+let programDataDir = ToMingwPath programDataDirWin
+let appDataLocalDirWin =  Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) |> EnsureWinDir
+let appDataLocalDir =  appDataLocalDirWin |> ToMingwPath
 
 let mbackupInstallDirWin = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles) |> EnsureDir |> fun s -> s + "mbackup"
 let mbackupInstallDir = mbackupInstallDirWin |> ToMingwPath
 
+let userHomeWin =
+  GetEnvDefault "HOME" (Environment.GetFolderPath(Environment.SpecialFolder.UserProfile))
+  |> EnsureWinDir
+
+let userHome = userHomeWin |> ToMingwPath
+
 //let userConfigDir = appDataRoamingDir + "mbackup/"
+let userConfigDirWin = programDataDirWin + "mbackup\\"
 let userConfigDir = programDataDir + "mbackup/"
+let runtimeDirWin = appDataLocalDirWin + "mbackup\\"
 let runtimeDir = appDataLocalDir + "mbackup/"
 
 let isLocalTarget (target: string) = target.StartsWith "/"
 
+// expand user file to mingw64 rsync supported path.
+// abc -> /cygdrive/c/Users/<user>/abc
+// ^Documents -> expand to Documents path.
+// ^Downloads -> expand to Downloads path.
+// etc
+let expandUserFile (fn: string) =
+  let fn =
+    let documentsDir = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) |> ToMingwPath |> EnsureDir
+    let picturesDir = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures) |> ToMingwPath |> EnsureDir
+    let desktopDir = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory) |> ToMingwPath |> EnsureDir
+    let fn = Regex.Replace(fn, "^My Documents/", documentsDir, RegexOptions.IgnoreCase)
+    let fn = Regex.Replace(fn, "^Documents/", documentsDir, RegexOptions.IgnoreCase)
+    let fn = Regex.Replace(fn, "^我的文档/", documentsDir)
+    let fn = Regex.Replace(fn, "^文档/", documentsDir)
+    let fn = Regex.Replace(fn, "^My Pictures/", picturesDir, RegexOptions.IgnoreCase)
+    let fn = Regex.Replace(fn, "^Pictures/", picturesDir, RegexOptions.IgnoreCase)
+    let fn = Regex.Replace(fn, "^图片/", picturesDir)
+    let fn = Regex.Replace(fn, "^Desktop/", desktopDir, RegexOptions.IgnoreCase)
+    let fn = Regex.Replace(fn, "^桌面/", desktopDir)
+    fn
+  if fn.StartsWith("/") then
+    fn
+  else
+    userHome + fn
+
 // generate mbackup.list file
-let generateMbackupList =
-    // TODO run make to update mbackup.list. only update if source file have changed.
-    null
+let generateMbackupList (logger: Logger) =
+    // TODO how to only regenerate if source file have changed? should I bundle GNU make with mbackup?
+    // just compare mbackup.list mtime with its source files?
+    let mbackupDefaultList = userConfigDirWin + "mbackup-default.list"
+    let mbackupLocalList = userConfigDirWin + "local.list"
+    let mbackupUserDefaultList = userConfigDirWin + "user-default.list"
+    let mbackupList = runtimeDirWin + "mbackup.list"
+
+    // local functions
+    let dropEmptyLinesAndComments = Seq.filter (fun (line: string) -> not (line.TrimStart().StartsWith("#") || line.TrimEnd().Equals("")))
+    let readMbackupListFile fn = File.ReadAllLines(fn) |> dropEmptyLinesAndComments
+
+    try
+      let defaultListLines = readMbackupListFile mbackupDefaultList |> Seq.map ToMingwPath
+      let localListLinesMaybe =
+        try
+          let lines = readMbackupListFile mbackupLocalList |> Seq.map ToMingwPath
+          (true, lines)
+        with
+          | :? System.IO.FileNotFoundException ->
+            (true, Seq.empty)
+          | ex ->
+            logger.Error "Read mbackupLocalList failed: %s" ex.Message
+            (false, Seq.empty)
+      match localListLinesMaybe with
+        | (false, _) -> failwith "Read mbackup local.list file failed"
+        | (true, localListLines) ->
+          let userDefaultListLines = readMbackupListFile mbackupUserDefaultList |> Seq.map expandUserFile
+          let allLines = Seq.append (Seq.append defaultListLines localListLines) userDefaultListLines
+          // For mbackup-default.list and local.list, exclude empty lines and comment lines.
+          // skip and give a warning on non-absolute path.
+          // For user-default.list, auto prefix user's home dir, auto expand Documents, Downloads etc special folder.
+          File.WriteAllLines(mbackupList, allLines)
+          logger.Info "%s written" mbackupList
+          true
+    with
+      | ex ->
+        logger.Error "Read/write mbackup list file failed: %s" ex.Message
+        false
 
 // append string s to list if pred is true
 let appendWhen (pred: bool) (lst: string list) (s: string) = if pred then List.append lst [s] else lst
@@ -128,8 +206,8 @@ let main argv =
 
     let logger = Logger()
 
-    logger.Info "userConfigDir=%s" userConfigDir
-    logger.Info "runtimeDir=%s" runtimeDir
+    logger.Info "using user config dir: %s" userConfigDirWin
+    logger.Info "using runtime dir: %s" runtimeDirWin
 
     let rsyncCmd: string list = []
     let rsyncCmd = appendWhen dryRun rsyncCmd "--dry-run"
@@ -137,7 +215,8 @@ let main argv =
     let rsyncCmd = List.append rsyncCmd ("-h --stats -togr --delete --delete-excluded --ignore-missing-args".Split [|' '|] |> Array.toList)
 
     let mbackupFile = runtimeDir + "mbackup.list"
-    generateMbackupList |> ignore
+    if not (generateMbackupList logger) then
+      failwith "Generate mbackup.list failed"
     let rsyncCmd = List.append rsyncCmd [sprintf "--files-from=%s" mbackupFile]
 
     let excludeFile = userConfigDir + "mbackup-default.exclude"
@@ -152,27 +231,21 @@ let main argv =
     let mbackupInstallDirWinTest = "D:\\downloads\\apps\\mbackupTest\\"
     let mbackupInstallDirTest = mbackupInstallDirWinTest |> ToMingwPath |> EnsureDir
     let sshExeFile = mbackupInstallDirTest + "rsync-w64/usr/bin/ssh.exe"
-    let sshConfigFile = userConfigDir + "ssh_config"
-    let sshPrivateKeyFile = userConfigDir + "ssh_id_rsa"
+    let sshConfigFile = userHome + ".ssh/config"
+    let sshPrivateKeyFile = results.GetResult(Ssh_Key, defaultValue = userHome + ".ssh/id_rsa") |> ToMingwPath
     let rsyncCmd = List.append rsyncCmd [sprintf "-e \"%s -F %s -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\"" sshExeFile sshConfigFile sshPrivateKeyFile]
 
     let backupTarget = results.GetResult (Target, defaultValue = Environment.GetEnvironmentVariable "TARGET")
     match backupTarget with
       | null ->
-          logger.Error "TARGET is not defined"
+          logger.Error "TARGET is not defined%s" ""
           ExitBadParam
       | _ ->
           let backupTarget = ToMingwPath backupTarget
           let rsyncCmd =
             if not (isLocalTarget backupTarget)
               then
-                let nodeName =
-                  match results.GetResult Node_Name with
-                    | null ->
-                      match GetEnv "NODE_NAME" with
-                        | null -> Net.Dns.GetHostName()
-                        | X -> X
-                    | X -> X
+                let nodeName = results.GetResult(Node_Name, defaultValue = Net.Dns.GetHostName())
                 let remoteLogFile = sprintf "/var/log/mbackup/%s.log" nodeName
                 let remoteUser = results.GetResult (Remote_User, defaultValue = Environment.UserName)
                 let rsyncCmd = List.append rsyncCmd [sprintf "--remote-option=--log-file=%s" remoteLogFile]
@@ -196,9 +269,9 @@ let main argv =
             if proc.WaitForExit Int32.MaxValue then
               proc.ExitCode
             else
-              logger.Error "mbackup timed out while waiting for rsync to complete"
+              logger.Error "mbackup timed out while waiting for rsync to complete%s" ""
               ExitTimeout
           with
             | _ ->
-              logger.Error "Create runtime dir failed"
+              logger.Error "Create runtime dir failed%s" ""
               ExitIOError

mbackup-config/mbackup-default.list

 # dirs to backup
-/cygdrive/c/ProgramData/mbackup
-/cygdrive/c/path/to/dir
-/cygdrive/d/path/to/dir
+C:\ProgramData\mbackup
+C:\path\to\dir
+D:\path\to\dir

mbackup-config/user-default.list

@@ -5,6 +5,11 @@
 #############################################
 Pictures/Saved Pictures/
 Documents/
+Desktop/
+
+#downloads dir is not supported yet. it will just be prefixed by user's profile dir.
+#if user moved this dir to other location, they should add dir to local.list.
+Downloads/
 
 #############################################
 # below are file list from linux environment

operational

@@ -6,6 +6,16 @@ Time-stamp: <2019-11-12>
 ** 2019-11-13 how to run it in dev env?
 dotnet run -- --dry-run --itemize-changes --target d:\backup
 dotnet run -- -n -i --target d:\backup
+dotnet run -- -n --target d:\backup
+
+try an ssh run:
+
+dotnet run -- -n --target root@sylecn01.emacsos.com:/data/backup/PC-backup/B75I3/
+it works.
+
+file list works.
+ssh transfer works.
+remote logging works.
 
 ** 2019-11-12 docs
 - rsync
@@ -44,9 +54,12 @@ dotnet run -- -n -i --target d:\backup
   MyPictures
   DesktopDirectory
   Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)
-- 
+- FSharpLint
+  http://fsprojects.github.io/FSharpLint/
+  http://fsprojects.github.io/FSharpLint/index.html
 
 * later                                                               :entry:
+** 2019-11-14 supports expand Downloads dir in user-default.list
 * current                                                             :entry:
 ** 
 ** 2019-11-13 extra user default list.
@@ -209,4 +222,5 @@ C:\ProgramData\mbackup\user-default.list
     it contains hostname.
   - 
 * done                                                                :entry:
+** 2019-11-13 build mbackup.list file from file list and exclude lists.
 * wontfix                                                             :entry: