Skip to content
GitLab
Explore
Sign in
Yuanle Song
mbackup-for-windows
Compare revisions
08723c365d1d49693dbb011c0d2ed0f12c2607d6 to d27c15d2aeec7458afc7563049c3a931bf75f352
Commits on Source (2)
generate mbackup.list works
· 466d2d83
Yuanle Song
authored
Nov 14, 2019
now rsync should work if ssh key is present.
466d2d83
rsync over ssh works.
· d27c15d2
Yuanle Song
authored
Nov 14, 2019
d27c15d2
Hide whitespace changes
Inline
Side-by-side
Program.fs
View file @
d27c15d2
...
...
@@ -11,8 +11,10 @@
module
Mbackup
open
System
open
System
.
IO
open
System
.
Diagnostics
open
System
.
Text
.
RegularExpressions
;
open
System
.
Diagnostics
.
CodeAnalysis
open
Argu
...
...
@@ -20,12 +22,14 @@ let ExitBadParam = 1
let
ExitTimeout
=
2
let
ExitIOError
=
3
[<
SuppressMessage
(
"*"
,
"UnionCasesNames"
)>]
type
CLIArguments
=
|
[<
AltCommandLine
(
"-n"
)>]
Dry_Run
|
Target
of
backupTarget
:
string
|
Remote_User
of
remoteUser
:
string
|
[<
AltCommandLine
(
"-i"
)>]
Itemize_Changes
|
Node_Name
of
nodeName
:
string
|
Ssh_Key
of
sshKeyFilename
:
string
with
interface
IArgParserTemplate
with
member
s
.
Usage
=
...
...
@@ -34,6 +38,8 @@ with
|
Target
_
->
"rsync target, could be local dir or remote ssh dir"
|
Remote_User
_
->
"remote linux user to own the backup files"
|
Itemize_Changes
_
->
"add -i option to rsync"
|
Node_Name
_
->
"local node's name, used in remote logging"
|
Ssh_Key
_
->
"ssh private key, used when backup to remote ssh node"
type
Logger
()
=
let
mutable
level
=
Logger
.
DEBUG
...
...
@@ -96,24 +102,96 @@ let ToMingwPath (windowsPath: string) =
result
let
EnsureDir
(
path
:
string
)
=
if
path
.
EndsWith
"/"
then
path
else
path
+
"/"
let
EnsureWinDir
(
path
:
string
)
=
if
path
.
EndsWith
"
\\
"
then
path
else
path
+
"
\\
"
let
appDataRoamingDir
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
ApplicationData
)
|>
ToMingwPath
|>
EnsureDir
let
programDataDir
=
GetEnv
"PROGRAMDATA"
|>
ToMingwPath
|>
EnsureDir
let
appDataLocalDir
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
LocalApplicationData
)
|>
ToMingwPath
|>
EnsureDir
let
programDataDirWin
=
GetEnv
"PROGRAMDATA"
|>
EnsureWinDir
let
programDataDir
=
ToMingwPath
programDataDirWin
let
appDataLocalDirWin
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
LocalApplicationData
)
|>
EnsureWinDir
let
appDataLocalDir
=
appDataLocalDirWin
|>
ToMingwPath
let
mbackupInstallDirWin
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
ProgramFiles
)
|>
EnsureDir
|>
fun
s
->
s
+
"mbackup"
let
mbackupInstallDir
=
mbackupInstallDirWin
|>
ToMingwPath
let
userHomeWin
=
GetEnvDefault
"HOME"
(
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
UserProfile
))
|>
EnsureWinDir
let
userHome
=
userHomeWin
|>
ToMingwPath
//let userConfigDir = appDataRoamingDir + "mbackup/"
let
userConfigDirWin
=
programDataDirWin
+
"mbackup
\\
"
let
userConfigDir
=
programDataDir
+
"mbackup/"
let
runtimeDirWin
=
appDataLocalDirWin
+
"mbackup
\\
"
let
runtimeDir
=
appDataLocalDir
+
"mbackup/"
let
isLocalTarget
(
target
:
string
)
=
target
.
StartsWith
"/"
// expand user file to mingw64 rsync supported path.
// abc -> /cygdrive/c/Users/<user>/abc
// ^Documents -> expand to Documents path.
// ^Downloads -> expand to Downloads path.
// etc
let
expandUserFile
(
fn
:
string
)
=
let
fn
=
let
documentsDir
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
MyDocuments
)
|>
ToMingwPath
|>
EnsureDir
let
picturesDir
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
MyPictures
)
|>
ToMingwPath
|>
EnsureDir
let
desktopDir
=
Environment
.
GetFolderPath
(
Environment
.
SpecialFolder
.
DesktopDirectory
)
|>
ToMingwPath
|>
EnsureDir
let
fn
=
Regex
.
Replace
(
fn
,
"^My Documents/"
,
documentsDir
,
RegexOptions
.
IgnoreCase
)
let
fn
=
Regex
.
Replace
(
fn
,
"^Documents/"
,
documentsDir
,
RegexOptions
.
IgnoreCase
)
let
fn
=
Regex
.
Replace
(
fn
,
"^我的文档/"
,
documentsDir
)
let
fn
=
Regex
.
Replace
(
fn
,
"^文档/"
,
documentsDir
)
let
fn
=
Regex
.
Replace
(
fn
,
"^My Pictures/"
,
picturesDir
,
RegexOptions
.
IgnoreCase
)
let
fn
=
Regex
.
Replace
(
fn
,
"^Pictures/"
,
picturesDir
,
RegexOptions
.
IgnoreCase
)
let
fn
=
Regex
.
Replace
(
fn
,
"^图片/"
,
picturesDir
)
let
fn
=
Regex
.
Replace
(
fn
,
"^Desktop/"
,
desktopDir
,
RegexOptions
.
IgnoreCase
)
let
fn
=
Regex
.
Replace
(
fn
,
"^桌面/"
,
desktopDir
)
fn
if
fn
.
StartsWith
(
"/"
)
then
fn
else
userHome
+
fn
// generate mbackup.list file
let
generateMbackupList
=
// TODO run make to update mbackup.list. only update if source file have changed.
null
let
generateMbackupList
(
logger
:
Logger
)
=
// TODO how to only regenerate if source file have changed? should I bundle GNU make with mbackup?
// just compare mbackup.list mtime with its source files?
let
mbackupDefaultList
=
userConfigDirWin
+
"mbackup-default.list"
let
mbackupLocalList
=
userConfigDirWin
+
"local.list"
let
mbackupUserDefaultList
=
userConfigDirWin
+
"user-default.list"
let
mbackupList
=
runtimeDirWin
+
"mbackup.list"
// local functions
let
dropEmptyLinesAndComments
=
Seq
.
filter
(
fun
(
line
:
string
)
->
not
(
line
.
TrimStart
()
.
StartsWith
(
"#"
)
||
line
.
TrimEnd
()
.
Equals
(
""
)))
let
readMbackupListFile
fn
=
File
.
ReadAllLines
(
fn
)
|>
dropEmptyLinesAndComments
try
let
defaultListLines
=
readMbackupListFile
mbackupDefaultList
|>
Seq
.
map
ToMingwPath
let
localListLinesMaybe
=
try
let
lines
=
readMbackupListFile
mbackupLocalList
|>
Seq
.
map
ToMingwPath
(
true
,
lines
)
with
|
:?
System
.
IO
.
FileNotFoundException
->
(
true
,
Seq
.
empty
)
|
ex
->
logger
.
Error
"Read mbackupLocalList failed: %s"
ex
.
Message
(
false
,
Seq
.
empty
)
match
localListLinesMaybe
with
|
(
false
,
_)
->
failwith
"Read mbackup local.list file failed"
|
(
true
,
localListLines
)
->
let
userDefaultListLines
=
readMbackupListFile
mbackupUserDefaultList
|>
Seq
.
map
expandUserFile
let
allLines
=
Seq
.
append
(
Seq
.
append
defaultListLines
localListLines
)
userDefaultListLines
// For mbackup-default.list and local.list, exclude empty lines and comment lines.
// skip and give a warning on non-absolute path.
// For user-default.list, auto prefix user's home dir, auto expand Documents, Downloads etc special folder.
File
.
WriteAllLines
(
mbackupList
,
allLines
)
logger
.
Info
"%s written"
mbackupList
true
with
|
ex
->
logger
.
Error
"Read/write mbackup list file failed: %s"
ex
.
Message
false
// append string s to list if pred is true
let
appendWhen
(
pred
:
bool
)
(
lst
:
string
list
)
(
s
:
string
)
=
if
pred
then
List
.
append
lst
[
s
]
else
lst
...
...
@@ -128,8 +206,8 @@ let main argv =
let
logger
=
Logger
()
logger
.
Info
"us
erConfigDir=
%s"
userConfigDir
logger
.
Info
"runtime
Dir=
%s"
runtimeDir
logger
.
Info
"us
ing user config dir:
%s"
userConfigDir
Win
logger
.
Info
"
using
runtime
dir:
%s"
runtimeDir
Win
let
rsyncCmd
:
string
list
=
[]
let
rsyncCmd
=
appendWhen
dryRun
rsyncCmd
"--dry-run"
...
...
@@ -137,7 +215,8 @@ let main argv =
let
rsyncCmd
=
List
.
append
rsyncCmd
(
"-h --stats -togr --delete --delete-excluded --ignore-missing-args"
.
Split
[|
'
'
|]
|>
Array
.
toList
)
let
mbackupFile
=
runtimeDir
+
"mbackup.list"
generateMbackupList
|>
ignore
if
not
(
generateMbackupList
logger
)
then
failwith
"Generate mbackup.list failed"
let
rsyncCmd
=
List
.
append
rsyncCmd
[
sprintf
"--files-from=%s"
mbackupFile
]
let
excludeFile
=
userConfigDir
+
"mbackup-default.exclude"
...
...
@@ -152,27 +231,21 @@ let main argv =
let
mbackupInstallDirWinTest
=
"D:
\\
downloads
\\
apps
\\
mbackupTest
\\
"
let
mbackupInstallDirTest
=
mbackupInstallDirWinTest
|>
ToMingwPath
|>
EnsureDir
let
sshExeFile
=
mbackupInstallDirTest
+
"rsync-w64/usr/bin/ssh.exe"
let
sshConfigFile
=
user
ConfigDir
+
"ssh
_
config"
let
sshPrivateKeyFile
=
userConfigDir
+
"ssh
_
id_rsa"
let
sshConfigFile
=
user
Home
+
"
.
ssh
/
config"
let
sshPrivateKeyFile
=
results
.
GetResult
(
Ssh_Key
,
defaultValue
=
userHome
+
"
.
ssh
/
id_rsa"
)
|>
ToMingwPath
let
rsyncCmd
=
List
.
append
rsyncCmd
[
sprintf
"-e
\"
%s -F %s -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
\"
"
sshExeFile
sshConfigFile
sshPrivateKeyFile
]
let
backupTarget
=
results
.
GetResult
(
Target
,
defaultValue
=
Environment
.
GetEnvironmentVariable
"TARGET"
)
match
backupTarget
with
|
null
->
logger
.
Error
"TARGET is not defined"
logger
.
Error
"TARGET is not defined
%s"
"
"
ExitBadParam
|
_
->
let
backupTarget
=
ToMingwPath
backupTarget
let
rsyncCmd
=
if
not
(
isLocalTarget
backupTarget
)
then
let
nodeName
=
match
results
.
GetResult
Node_Name
with
|
null
->
match
GetEnv
"NODE_NAME"
with
|
null
->
Net
.
Dns
.
GetHostName
()
|
X
->
X
|
X
->
X
let
nodeName
=
results
.
GetResult
(
Node_Name
,
defaultValue
=
Net
.
Dns
.
GetHostName
()
)
let
remoteLogFile
=
sprintf
"/var/log/mbackup/%s.log"
nodeName
let
remoteUser
=
results
.
GetResult
(
Remote_User
,
defaultValue
=
Environment
.
UserName
)
let
rsyncCmd
=
List
.
append
rsyncCmd
[
sprintf
"--remote-option=--log-file=%s"
remoteLogFile
]
...
...
@@ -196,9 +269,9 @@ let main argv =
if
proc
.
WaitForExit
Int32
.
MaxValue
then
proc
.
ExitCode
else
logger
.
Error
"mbackup timed out while waiting for rsync to complete"
logger
.
Error
"mbackup timed out while waiting for rsync to complete
%s"
"
"
ExitTimeout
with
|
_
->
logger
.
Error
"Create runtime dir failed"
logger
.
Error
"Create runtime dir failed
%s"
"
"
ExitIOError
mbackup-config/mbackup-default.list
View file @
d27c15d2
# dirs to backup
/cygdrive/c/
ProgramData
/
mbackup
/cygdrive/c/
path
/
to
/
dir
/cygdrive/d/
path
/
to
/
dir
C:\
ProgramData
\
mbackup
C:\
path
\
to
\
dir
D:\
path
\
to
\
dir
mbackup-config/user-default.list
View file @
d27c15d2
...
...
@@ -5,6 +5,11 @@
#############################################
Pictures/Saved Pictures/
Documents/
Desktop/
#downloads dir is not supported yet. it will just be prefixed by user's profile dir.
#if user moved this dir to other location, they should add dir to local.list.
Downloads/
#############################################
# below are file list from linux environment
...
...
operational
View file @
d27c15d2
...
...
@@ -6,6 +6,16 @@ Time-stamp: <2019-11-12>
** 2019-11-13 how to run it in dev env?
dotnet run -- --dry-run --itemize-changes --target d:\backup
dotnet run -- -n -i --target d:\backup
dotnet run -- -n --target d:\backup
try an ssh run:
dotnet run -- -n --target root@sylecn01.emacsos.com:/data/backup/PC-backup/B75I3/
it works.
file list works.
ssh transfer works.
remote logging works.
** 2019-11-12 docs
- rsync
...
...
@@ -44,9 +54,12 @@ dotnet run -- -n -i --target d:\backup
MyPictures
DesktopDirectory
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)
-
- FSharpLint
http://fsprojects.github.io/FSharpLint/
http://fsprojects.github.io/FSharpLint/index.html
* later :entry:
** 2019-11-14 supports expand Downloads dir in user-default.list
* current :entry:
**
** 2019-11-13 extra user default list.
...
...
@@ -209,4 +222,5 @@ C:\ProgramData\mbackup\user-default.list
it contains hostname.
-
* done :entry:
** 2019-11-13 build mbackup.list file from file list and exclude lists.
* wontfix :entry: