.gitignore

@@ -7,3 +7,4 @@ pypi/rd-client/rdclient/rd
 pypi/rd-api/rdapi/rd-api
 *README.rst
 stack.yaml.lock
+*.whl

README.md

@@ -14,11 +14,11 @@ rd-api - reliable download server
 
 Usage: rd-api [-h|--host HOST] [-p|--port PORT] [--redis-host REDIS_HOST]
               [--redis-port REDIS_PORT] [-d|--web-root DIR] [-w|--worker INT]
-              [-V|--version]
+              [-v|--verbose] [-V|--version]
   rd-api is an HTTP file server that provides static file hosting and reliable
   download api for rd client.
   
-  rd-api serves files under web-root. You can use it like python3 -m http.server
+  rd-api serves files under web-root. You can use it like ```python3 -m http.server```
   
   In addition, if rd command line tool is used to do the download, it will
   download in a reliable way by downloading in 2MiB blocks and verify checksum
@@ -54,6 +54,7 @@ Available options:
   -d,--web-root DIR        web root directory (default: ".")
   -w,--worker INT          how many concurrent workers to calculator sha1sum for
                            file (default: 2)
+  -v,--verbose             show more debug message
   -V,--version             show program version and exit
   -h,--help                Show this help text
 
@@ -61,7 +62,7 @@ Available options:
 
 ```
 $ rd --help
-rd - reliable download command line tool
+rd - reliable download client
 
 Usage: rd [-r|--block-max-retry INT] [-k|--keep] [-d|--temp-dir TEMP_DIR]
           [-o|--output-dir OUTPUT_DIR] [-w|--worker INT] [-f|--force]

api/Main.hs

@@ -19,7 +19,6 @@ import qualified Text.PrettyPrint.ANSI.Leijen as D
 
 import Config
 import CliVersion (cliVersion)
-import Utils
 import Opts (argParser)
 import OptsDoc (rdApiDescription)
 import App (mkWaiApp)
@@ -68,7 +67,8 @@ runApiServer rdConfig = do
   configE <- liftIO $ updateRDConfigFromEnv rdConfig
   configMaybe <- case configE of
     Left e -> do
-      liftIO $ logl rc0 $ sformat ("Error: " % stext) e
+      -- liftIO $ L.err (rcLogger rc0) $ L.msg $ sformat ("Error: " % stext) e
+      liftIO $ errorl rc0 $ sformat ("Error: " % stext) e
       mzero    -- early exit
     Right config -> return $ Just config
   let config = fromJust configMaybe
@@ -79,10 +79,10 @@ runApiServer rdConfig = do
   connMaybe <- case connEi of
     Left e -> do
       liftIO $ do
-        logl rc0 $ sformat
+        errorl rc0 $ sformat
           ("Connect to redis at " % string % ":" % int % " failed: " % stext)
           (redisHost config) (redisPort config) e
-        logl rc0 $ sformat "No redis, GET /rd/ api disabled, acting as static file server"
+        warnl rc0 $ sformat "No redis, GET /rd/ api disabled, acting as static file server"
       return Nothing
     Right conn ->
       return $ Just conn
@@ -96,9 +96,9 @@ runApiServer rdConfig = do
     if rcHasRedis rc then
         startWorkers rc
     else
-        logl rc $ sformat "No redis, not starting workers"
-    logl rc $ sformat ("webRoot is " % string) (webRoot config)
-    logl rc $ sformat ("will listen on " % string % ":" % int) (host config) (port config)
+        warnl rc $ sformat "No redis, not starting workers"
+    infol rc $ sformat ("webRoot is " % string) (webRoot config)
+    infol rc $ sformat ("will listen on " % string % ":" % int) (host config) (port config)
     let warpSettings = ( setFdCacheDuration 10
                        . setFileInfoCacheDuration 10
                        . setPort (port config)

api/Opts.hs

@@ -46,6 +46,11 @@ argParser = RDConfig
       <> showDefault
       <> value 2
       <> metavar "INT")
+  <*> switch
+      (  long "verbose"
+      <> short 'v'
+      <> help "show more debug message"
+      <> showDefault )
   <*> switch
       (  long "version"
       <> short 'V'

src/App.hs → api/lib/App.hs

@@ -7,7 +7,7 @@ import Data.Either.Extra (fromRight')
 import Data.Text.Encoding (decodeUtf8)
 import Control.Concurrent.Chan
 import System.IO.Error (catchIOError)
-import Control.Monad (when, unless)
+import Control.Monad (unless)
 import qualified Data.Text as T
 import qualified Data.Text.Lazy as LT
 
@@ -33,11 +33,11 @@ fillSha1sum rc fbp = do
   redisReply <- R.runRedis (rcRedisConn rc) $ R.hgetall hashKey
   case redisReply of
     Left reply -> do
-      logl rc $ "redis hgetall " <> showt hashKey <> " failed: " <> showt reply
+      errorl rc $ "redis hgetall " <> showt hashKey <> " failed: " <> showt reply
       return $ map fillBlock (fbpBlocks fbp) where
         fillBlock (blockId, start, end) = (blockId, start, end, "pending")
     Right blockIdSha1sumAlist -> do
-      logl rc $ "fillSha1sum: redis hgetall " <> showt hashKey <> " ok"
+      debugl rc $ "fillSha1sum: redis hgetall " <> showt hashKey <> " ok"
       return $ map fillBlock (fbpBlocks fbp) where
         blockIdSha1sumMap = M.fromList blockIdSha1sumAlist
         fillBlock :: Block -> BlockWithChecksum
@@ -48,26 +48,31 @@ fillSha1sum rc fbp = do
 processNewFileAsyncMaybe :: RDRuntimeConfig -> FillBlockParam -> ExceptT T.Text IO ()
 processNewFileAsyncMaybe rc fbp = do
   let strKey = fileStatusKey fbp
-  resultE <- liftIO $ DB.insertIfNotExist rc strKey fileStatusWorking
+      filePath = fbpFilepath fbp
+  resultE <- liftIO $ DB.insertIfNotExist rc strKey $ fsBytes FileStatusWorking
   throwOnLeft resultE
   let insertOk = fromRight' resultE
   if insertOk then liftIO $ do
-      logl rc $ showt strKey <> " is a new file, sending task to worker"
+      infol rc $ showt filePath <> " is a new file, sending task to worker"
       writeChan (rcFileChan rc) fbp
       return ()
   else do
     oldStatusE <- liftIO $ do
-      logl rc $ showt strKey <> " is not a new file"
+      debugl rc $ showt filePath <> " is not a new file"
       -- if status is error, set it to working, then add task to fileChan
       DB.get rc strKey
     throwOnLeftMsg oldStatusE "get old file status failed"
     let oldStatus = fromRight' oldStatusE
-    when (oldStatus == Just fileStatusError) $ do
+    case fmap fsFromBytes oldStatus of
+      Just FileStatusError -> do
         setResultE <- liftIO $ do
-          logl rc $ showt strKey <> " was in " <> showt fileStatusError <> " status"
-          DB.set rc strKey fileStatusWorking
-        throwOnLeftMsg setResultE $ "set file status to " <> showt fileStatusWorking <> " failed"
+          infol rc $ showt strKey <> " was in " <> showt FileStatusError <> " status"
+          DB.set rc strKey $ fsBytes FileStatusWorking
+        throwOnLeftMsg setResultE $ "set file status to " <> showt FileStatusWorking <> " failed"
         liftIO $ writeChan (rcFileChan rc) fbp
+      Just FileStatusDone -> liftIO $ infol rc "file was processed before"
+      Just FileStatusWorking -> liftIO $ infol rc "file is being processed by worker"
+      _ -> liftIO $ errorl rc "Unexpected file status"
 
 -- | GET /rd/.* handler
 getRdHandler :: RDRuntimeConfig -> ExceptT T.Text ActionM ()
@@ -78,12 +83,12 @@ getRdHandler rc = do
 
   let filepath = webRoot (rcConfig rc) </> T.unpack path
   fileStatusE <- lift $ do
-    liftIO $ logl rc $ "user request " <> showt filepath
+    liftIO $ infol rc $ "user request " <> showt filepath
     liftIO $ catchIOError
       (fmap Right (getFileStatus filepath))
       (\e -> do
          let msg = "getFileStatus on " <> T.pack filepath <> " failed"
-         logl rc $ msg <> ":\n\t" <> T.pack (show e)
+         errorl rc $ msg <> ":\n\t" <> T.pack (show e)
          return $ Left msg)
   throwOnLeft fileStatusE
   let fileStatus = fromRight' fileStatusE

src/Config.hs → api/lib/Config.hs

@@ -4,8 +4,10 @@ import qualified Database.Redis as R
 import Control.Concurrent.Chan
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as Char8
+import qualified Data.Text as T
+import Control.Monad.IO.Class
 
-import System.Log.FastLogger
+import qualified System.Logger as L
 
 import Type
 
@@ -17,6 +19,7 @@ data RDConfig = RDConfig {
     , redisPort :: Int
     , webRoot :: FilePath
     , fileWorkerCount :: Int
+    , verbose :: Bool
     , showVersion :: Bool } deriving (Show)
 
 data RDRuntimeConfig = RDRuntimeConfig {
@@ -24,8 +27,31 @@ data RDRuntimeConfig = RDRuntimeConfig {
     , rcRedisConn :: R.Connection
     , rcHasRedis :: Bool
     , rcFileChan :: Chan FillBlockParam
-    , rcLoggerSet :: LoggerSet
-    , rcLoggerTimeCache :: IO FormattedTime }
+    , rcLogger :: L.Logger }
+
+errorl :: MonadIO m => RDRuntimeConfig -> T.Text -> m ()
+errorl rc msg = do
+  let logger = rcLogger rc
+  L.err logger $ L.msg msg
+  L.flush logger
+
+warnl :: MonadIO m => RDRuntimeConfig -> T.Text -> m ()
+warnl rc msg = do
+  let logger = rcLogger rc
+  L.warn logger $ L.msg msg
+  L.flush logger
+
+infol :: MonadIO m => RDRuntimeConfig -> T.Text -> m ()
+infol rc msg = do
+  let logger = rcLogger rc
+  L.info logger $ L.msg msg
+  L.flush logger
+
+debugl :: MonadIO m => RDRuntimeConfig -> T.Text -> m ()
+debugl rc msg = L.debug (rcLogger rc) $ L.msg msg
+
+flushl :: MonadIO m => RDRuntimeConfig -> m ()
+flushl rc = L.flush (rcLogger rc)
 
 defaultRDConfig :: RDConfig
 defaultRDConfig = RDConfig {
@@ -35,6 +61,7 @@ defaultRDConfig = RDConfig {
                   , redisPort = 6379
                   , webRoot = "/nonexistent"
                   , fileWorkerCount = 2
+                  , verbose = False
                   , showVersion = False
                   }
 
@@ -42,15 +69,17 @@ defaultRDRuntimeConfig :: RDConfig -> IO RDRuntimeConfig
 defaultRDRuntimeConfig config = do
   conn <- R.connect R.defaultConnectInfo
   fileChan <- newChan
-  loggerSet <- newStdoutLoggerSet defaultBufSize
-  loggerTimeCache <- newTimeCache simpleTimeFormat
+  let logLevel = if verbose config then L.Debug else L.Info
+      logSettings = (L.setFormat (Just "%Y-%0m-%0dT%0H:%0M:%0S") .
+                     L.setLogLevel logLevel .
+                     L.setDelimiter "  ") L.defSettings
+  logger <- L.new logSettings
   return RDRuntimeConfig {
                rcConfig=config
              , rcRedisConn=conn
              , rcHasRedis=True
              , rcFileChan=fileChan
-             , rcLoggerSet=loggerSet
-             , rcLoggerTimeCache=loggerTimeCache }
+             , rcLogger=logger }
 
 -- | the redis hash key used to store cached sha1sum for given FillBlockParam
 blockSha1sumHashKey :: FillBlockParam -> B.ByteString

src/DB.hs → api/lib/DB.hs

@@ -13,8 +13,7 @@ insertIfNotExist rc key value = do
   redisReply <- R.runRedis (rcRedisConn rc) $ R.setnx key value
   case redisReply of
     Left reply -> do
-      let msg = "redis setnx " <> showt key <> " failed:\n\t" <> showt reply
-      logl rc msg
+      errorl rc $ "redis setnx " <> showt key <> " failed:\n\t" <> showt reply
       return $ Left "insertIfNotExist on DB failed"
     Right v -> return $ Right v
 
@@ -24,7 +23,7 @@ get rc key = do
   case redisReply of
     Left reply -> do
       let msg = "redis get " <> showt key <> " failed: " <> showt reply
-      logl rc msg
+      errorl rc msg
       return $ Left msg
     Right v -> return $ Right v
 
@@ -34,6 +33,6 @@ set rc key value = do
   case redisReply of
     Left reply -> do
       let msg = "redis set " <> showt key <> " to " <> showt value <> " failed: " <> showt reply
-      logl rc msg
+      errorl rc msg
       return $ Left msg
     Right v -> return $ Right v

src/Worker.hs → api/lib/Worker.hs

@@ -39,23 +39,23 @@ sha1sumFileRange filepath start end = sha1sumOnBytes <$> fileRange filepath star
 -- for all blocks and write result to redis. then mark the file as done.
 fileWorker :: RDRuntimeConfig -> IO ()
 fileWorker rc = forever $ do
-  logl rc ("fileWorker is waiting for jobs..." :: T.Text)
+  infol rc ("fileWorker is waiting for jobs..." :: T.Text)
   fbp <- readChan (rcFileChan rc)
   let filepath = fbpFilepath fbp
       conn = rcRedisConn rc
   -- calculate sha1sum for each block and write result to redis hash
-  logl rc $ "fileWorker working on " <> showt filepath
+  infol rc $ "fileWorker working on " <> showt filepath
   results <- withBinaryFile filepath ReadMode $ \handle -> do
     let hashKey = blockSha1sumHashKey fbp
     mapM (calculateSha1ForBlock conn hashKey handle) (fbpBlocks fbp)
-  let resultStatus = if and results then fileStatusDone else fileStatusError
-  redisReply <- R.runRedis conn $ R.set (fileStatusKey fbp) resultStatus
+  let resultStatus = if and results then FileStatusDone else FileStatusError
+  redisReply <- R.runRedis conn $ R.set (fileStatusKey fbp) $ fsBytes resultStatus
   case redisReply of
     Left reply ->
-      logl rc $ "set file status failed: " <> showt reply
+      errorl rc $ "Set file status failed: " <> showt reply
     Right _ -> do
-      logl rc $ "set file status to " <> showt resultStatus <> " for " <> showt filepath
-      logl rc $ "fileWorker done for " <> showt filepath
+      debugl rc $ "Set file status to " <> showt resultStatus <> " for " <> showt filepath
+      infol rc $ "fileWorker done for " <> showt filepath
   return ()
     where
       -- | calculate sha1 for a single block. return IO True on success.
@@ -64,12 +64,12 @@ fileWorker rc = forever $ do
         redisReply <- R.runRedis conn $ R.hget hashKey (blockIdKey blockId)
         case redisReply of
           Left reply -> do
-            logl rc $ "redis hget failed on " <> showt hashKey <> ": " <> showt reply
+            errorl rc $ "redis hget failed on " <> showt hashKey <> ": " <> showt reply
             return False
           Right sha1sumMaybe ->
             case sha1sumMaybe of
               Just _sha1sum -> do
-                logl rc $ "skip calculated block " <> showt blockId
+                debugl rc $ "skip calculated block " <> showt blockId
                 return True
               Nothing -> do
                 blockContent <- fileRangeH handle start end
@@ -77,12 +77,12 @@ fileWorker rc = forever $ do
                 redisReply2 <- R.runRedis conn $ R.hset hashKey (blockIdKey blockId) blockSha1
                 case redisReply2 of
                   Left reply -> do
-                    logl rc $ "redis hset failed on " <> showt hashKey <> ": " <> showt reply
+                    errorl rc $ "redis hset failed on " <> showt hashKey <> ": " <> showt reply
                     return False
                   Right n ->
                     if n > 0 then
                       do
-                        logl rc $ "redis hset " <> showt hashKey <> " " <> showt blockId <> " ok"
+                        debugl rc $ "redis hset " <> showt hashKey <> " " <> showt blockId <> " ok"
                         return True
                      else
                         return False
@@ -90,5 +90,5 @@ fileWorker rc = forever $ do
 startWorkers :: RDRuntimeConfig -> IO ()
 startWorkers rc = do
   let workerCount = fileWorkerCount $ rcConfig rc
-  logl rc $ "creating " <> showt workerCount <> " file worker(s)"
+  infol rc $ "creating " <> showt workerCount <> " file worker(s)"
   replicateM_ workerCount (forkIO $ fileWorker rc)

src/CliVersion.hs → cli-version/CliVersion.hs

 module CliVersion where
 
 cliVersion :: String
-cliVersion = "1.1.0.0"
+cliVersion = "1.1.3.0"

client/Main.hs

@@ -178,20 +178,20 @@ combineBlocks rc rdResp = do
           removeFile targetFilename
           return True)
         (\e -> do
-           errorl rc $ "remove existing file failed: " <> showt e
+           errorl rc $ "Remove existing file failed: " <> showt e
            return False)
       unless result mzero
   liftIO $ do
-    infol rc $ "combining blocks to create " <> showt targetFilename
+    infol rc $ "Combining blocks to create " <> showt targetFilename
     forM_ (getBlockTargetFilenames opts rdResp) $ \blockFilename -> do
       debugl rc $ "appending block file " <> showt blockFilename
       content <- LB.readFile blockFilename
       LB.appendFile targetFilename content  -- TODO how to handle error here?
                                             -- let it crash?
-    infol rc $ "file downloaded to " <> showt targetFilename
+    infol rc $ "File downloaded to " <> showt targetFilename
     unless (keepBlockData opts) $ do
       let tempdir = tempDir opts </> filename
-      debugl rc $ "delete temporary block data dir " <> showt tempdir
+      debugl rc $ "Delete temporary block data dir " <> showt tempdir
       catchIOError (removeDirectoryRecursive tempdir)
                    (\e -> warnl rc $ "Warning: delete temp block data dir failed: " <> showt e)
 
@@ -276,7 +276,7 @@ cliApp :: RDOptions -> IO ()
 cliApp opts = do
   let level = if verbose opts then L.Debug else L.Info
       -- Note: tinylog doesn't support non-GMT dateformat.
-      logSettings = (L.setFormat (Just "%0H:%0M:%0S") .
+      logSettings = (L.setFormat (Just "%Y-%0m-%0dT%0H:%0M:%0S") .
                      L.setLogLevel level .
                      L.setDelimiter "  ")
                     L.defSettings
@@ -294,7 +294,7 @@ cliApp opts = do
   resultsMaybe <- mapM (runMaybeT . downloadFile rc) (urls opts)
   let results = map (fromMaybe False) resultsMaybe
   if and results then
-      infol rc "all urls downloaded."
+      infol rc "All urls downloaded."
   else do
       errorl rc $ (showt . length . filter not) results <> " urls failed/skipped."
       exitFailure
@@ -313,5 +313,5 @@ main = do
   where
     parserInfo = info (argParser <**> helper)
       (  fullDesc
-      <> header "rd - reliable download command line tool"
+      <> header "rd - reliable download client"
       <> progDesc "Download large files across slow and unstable network reliably. Requires using rd-api on server side. For more information, see rd-api --help")

src/Type.hs → lib/Type.hs

 module Type where
 
+import qualified Data.ByteString.Char8 as Char8
 import qualified Data.ByteString as B
 import qualified Data.Text as T
 
@@ -17,14 +18,29 @@ getBlockSha1sum (_, _, _, sha1sum) = sha1sum
 getBlockId :: BlockWithChecksum -> BlockID
 getBlockId (blockId, _, _, _) = blockId
 
-fileStatusWorking :: B.ByteString
-fileStatusWorking = "working"
+data FileStatus
+    = FileStatusWorking
+    | FileStatusError
+    | FileStatusDone
+    | FileStatusUnknown    -- unexpected/unknown status
+    deriving (Eq)
 
-fileStatusError :: B.ByteString
-fileStatusError = "error"
+instance Show FileStatus where
+    show FileStatusWorking = "working"
+    show FileStatusError = "error"
+    show FileStatusDone = "done"
+    show FileStatusUnknown = "unknown"
 
-fileStatusDone :: B.ByteString
-fileStatusDone = "done"
+fsBytes :: FileStatus -> B.ByteString
+fsBytes = Char8.pack . show
+
+fsFromBytes :: B.ByteString -> FileStatus
+fsFromBytes bs =
+    case bs of
+      "working" -> FileStatusWorking
+      "error" -> FileStatusError
+      "done" -> FileStatusDone
+      _ -> FileStatusUnknown
 
 data FillBlockParam = FillBlockParam {
       fbpFilepath :: FilePath

src/Utils.hs → lib/Utils.hs

@@ -5,20 +5,11 @@ module Utils where
 import qualified Data.Text as T
 
 import Control.Error
-import System.Log.FastLogger
-
-import Config
 
 -- | like show, but return a T.Text
 showt :: Show a => a -> T.Text
 showt = T.pack . show
 
--- | log a message that implements ToLogStr
-logl :: ToLogStr a => RDRuntimeConfig -> a -> IO ()
-logl rc msg = do
-  ts <- rcLoggerTimeCache rc
-  pushLogStrLn (rcLoggerSet rc) $ toLogStr ts <> " " <> toLogStr msg
-
 -- | signal an exception if given eitherValue is a Left.
 throwOnLeft :: Monad m => Either T.Text a -> ExceptT T.Text m ()
 throwOnLeft eitherValue =

misc/logtest/Main.hs

+-- The ReaderT Design Pattern
+-- https://www.fpcomplete.com/blog/2017/06/readert-design-pattern/
+
+module Main (main) where
+
+import Control.Monad.Trans.Reader
+
+import qualified System.Logger as L
+
+data LogTestOptions = LogTestOptions
+    { verbose :: Bool
+    , workerCount :: Int } deriving (Show)
+
+data LogTestRuntimeConfig = LogTestRuntimeConfig
+                          { logTestOptions :: LogTestOptions
+                          , logTestLogger :: L.Logger }
+
+type LogTestApp = ReaderT LogTestRuntimeConfig IO
+
+parseConfig :: IO LogTestRuntimeConfig
+parseConfig = do
+  let logSettings = (L.setFormat (Just "%Y-%0m-%0dT%0H:%0M:%0S") .
+                     L.setLogLevel L.Info .
+                     L.setDelimiter "  ")
+                    L.defSettings
+  logger <- L.new logSettings
+  return LogTestRuntimeConfig
+             { logTestOptions=LogTestOptions
+                                 { verbose=True
+                                 , workerCount=2}
+             , logTestLogger=logger}
+
+step1 :: LogTestApp ()
+step1 = do
+  rc <- ask
+  let logger = logTestLogger rc
+  L.info logger $ L.msg $ L.val "Running step1.1"
+  L.info logger $ L.msg $ L.val "Running step1.2"
+  L.flush logger
+
+step2 :: LogTestApp ()
+step2 = do
+  rc <- ask
+  let logger = logTestLogger rc
+  L.info logger $ L.msg $ L.val "Running step2"
+  L.flush logger
+
+main :: IO ()
+main = do
+  rc <- parseConfig
+  runReaderT
+    (do
+      step1
+      step2)
+    rc

operational

 * COMMENT -*- mode: org -*-
 #+Date: 2018-05-04
-Time-stamp: <2022-03-12>
+Time-stamp: <2022-03-14>
 #+STARTUP: content
 * notes                                                               :entry:
-** 2018-05-09 how to release latest code on PyPI?
+** 2022-03-14 project dir structure
+- lib      shared code for server and client, the reliable-download library
+- api      rd-api cli Main
+- api/lib  rd-api cli logic
+- client   rd cli Main
+- cli-version    cli-version string shared by rd-api and rd cli Main
+- misc/    learning tools and temp codes
+- test/    tests
+
+** 2018-05-09 how to release latest code on PyPI? how to make a release?
 - update version number in package.yaml, src/CliVersion.hs
 - build binary
-  stack build --test
+  stack build --test --pedantic
   stack exec hlint -- -g
-- update README file
+- update README file. add ChangeLog entry on *README.rst
+  README.md
+  rd-api-README.rst
+  rd-README.rst
 - release binary on PyPI
   export TWINE_PASSWORD=xxx
   make all -C pypi
@@ -83,6 +95,10 @@ Time-stamp: <2022-03-12>
   curl -XGET http://localhost:8082/rd/ideaIC-2018.1.tar.gz
   curl -XGET http://localhost:8082/rd/ideaIC-2018.1.tar.gz | jq .
 
+  To clear cached file status for ideaIC-2018.1.tar.gz,
+  redis-cli del "/home/sylecn/persist/cache/ideaIC-2018.1.tar.gz_2097152_status"
+  the 2097152 there is 2MiB block size in bytes.
+
 - client tool:
   cd ~/projects/reliable-download/
   curl http://localhost:8082/rd/ideaIC-2018.1.tar.gz
@@ -157,7 +173,10 @@ this should return json of the block metadata.
   - 
 
 ** 2018-05-06 how to run hlint
-stack exec hlint -- src api client
+stack exec hlint -- src api client logtest
+
+or run on all git files:
+stack exec hlint -- -g
 ** 2018-05-05 it's impossible to do logging easily in haskell.
 two problems
 
@@ -295,6 +314,13 @@ https://artyom.me/aeson
 ** 2018-05-06 optparse-applicative :: Stackage Server
 https://www.stackage.org/lts-10.3/package/optparse-applicative-0.14.0.0
 * later                                                               :entry:
+** 2022-03-12 drop redis-server as rd-api dependency.
+- use a built-in key-value db. such as Berkeley db, sqlite3, or leveldb.
+  use a well known path for the db name.
+
+  $HOME/.cache/reliable-downloader/rd-api.db
+- 
+
 ** 2018-05-07 loopUntilAllBlocksReady, how to track progress?
 use a thread pool to download blocks, print overall progress when some parts
 done or some time elapsed.
@@ -499,6 +525,59 @@ only first character is in path key.
 
 * current                                                             :entry:
 ** 
+** 2022-03-14 can I make tinylog log timestamp use localtime?
+** 2022-03-14 release v1.1.3.0
+- v1.1.3.0 changes
+  - revised logging messages
+  - rd-api support --verbose option. debug msg is not shown by default.
+  - code ported to ghc 8.10.7
+
+- v1.1.4.0 changes
+  - TODO IO error handling when combine blocks to big file
+
+- problems
+  - DONE CliVersion module can be moved away from library.
+    it is only used in rd client and rd-api executable.
+
+    seems not worth the trouble.
+
+    search: haskell package.yaml extra source file
+
+    source-dirs is a list.
+
+  - DONE why does rd-client rely on library?
+    rd-api and rd has some shared code in Utils.hs and Lib.hs
+
+    move non-shared code to api/ module.
+
+  - DONE how to test api module after some files moved from reliable-download
+    library?
+
+    move api library code to api/lib/ dir.
+    include that source dir in test binary.
+
+    it works.
+
+  - TODO do I need to build on older system so pypi version will work on all
+    OS? e.g. debian 9 or debian 10.
+
+    try run binary built on debian 11 in debian 10 VM.
+
+** 2021-10-07 When combine blocks to final file, do not use more disk space than original file size.
+One way is, open target file in write mode, append block 0, remove block 0.
+Append block 1, remove block 1.
+...
+Until last block.
+
+But this will fail if there is a disk failure.
+
+Maybe only do this when local disk is low on space.
+
+Is there an API that atomically combine two files? Reuse existing disk sectors
+of the two files.
+
+- What's the current combine logic?
+
 ** 2019-02-28 bug: rd-api -d java/
 option -d: cannot parse value `java/'
 
@@ -529,6 +608,168 @@ policy in ovs can do it.
   see stretch01 daylog.
 
 * done                                                                :entry:
+** 2022-03-12 improve logging on rd-api and rd client.
+- For big files, when there is a lot of blocks, rd-api default log is too
+  much.
+
+  default INFO level log should be easy to read.
+  just enough for the user to understand what's going on.
+- on client side, "block xx fetched" is not as useful as "xx/xx block fetched,
+  xx%"
+- implementation
+  ./src/App.hs
+  mkApp
+
+  getRdHandler
+
+- DONE client side log should have datatime as ts, not just time.
+  use the default ISO8601 UTC time.
+  it doesn't support local time.
+
+  client side use tinylog
+  import qualified System.Logger as L
+  and a few wrappers, such as debugl, infol, warnl etc.
+  rdLogger is saved in RDClientRuntimeConfig.
+
+  tinylog last release is from 2019.5 no updates since then.
+
+  other logging systems
+
+  katip: A structured logging framework.
+  https://hackage.haskell.org/package/katip
+  too many dependencies for a logging library.
+
+  simple-logger: A very simple but efficient logging framework
+  https://hackage.haskell.org/package/simple-logger
+  last release 2020.12
+
+  heavy-logger: Full-weight logging based on fast-logger
+  https://hackage.haskell.org/package/heavy-logger
+
+  co-log :: Kowainik
+  https://kowainik.github.io/projects/co-log
+  co-log/tutorials at main · co-log/co-log
+  https://github.com/co-log/co-log/tree/main/tutorials
+  still too complicated because I don't use monad transformers in my app.
+
+  How do I do logging in Haskell? - Stack Overflow
+  https://stackoverflow.com/questions/6310961/how-do-i-do-logging-in-haskell
+
+  just continue to use tinylog.
+
+- ~/projects/reliable-download/src/Utils.hs
+  rd-api use fast-logger
+  import System.Log.FastLogger
+
+  fast-logger: A fast logging system
+  https://hackage.haskell.org/package/fast-logger-3.1.1
+  last release is 2022.1, good.
+
+  -- | log a message that implements ToLogStr
+  logl :: ToLogStr a => RDRuntimeConfig -> a -> IO ()
+
+  I can implement log level here myself.
+
+  logl rc $
+
+  try convert from fast-logger to tinylog.
+  api/Main.hs
+  src/App.hs
+  src/DB.hs
+  src/Utils.hs    drop logl definition.
+  src/Worker.hs
+
+- test logging works as expected.
+  without --verbose
+  with --verbose
+
+  rd-api looks good.
+  rd looks okay. progress tracking can be further improved.
+
+  - problems
+    - MOVED can I make log timestamp use localtime?
+
+    - why binary is so huge?
+      ll "/home/sylecn/projects/reliable-download/.stack-work/install/x86_64-linux-tinfo6/9d918e84fe7a0b1a34181819ba567c0f47f721374dbb8070c76ed2c65393d9b2/8.10.7/bin"
+      23M rd
+      24M rd-api
+
+      it's already stripped.
+
+      old release size
+      binary: rd-api 19M, rd 22M
+      whl file: rd-api 4M, rd 4.7M
+
+      so zip can compress most of it.
+      I still want to reduce binary size though.
+
+      search: haskell stack reduce binary size
+
+      is ghc -O2 enabled by default? yes.
+
+- problems
+  - rd client has log level. api doesn't have log level.
+
+    I need to add log level to api.
+
+    fillSha1sum: redis hgetall "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152" ok
+
+    redis hset "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152" 0 ok
+
+    set file status to "done" for "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl"
+
+    these should be debug log.
+
+  - DONE 12/Mar/2022:23:01:15 +0800 "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152_status" is not a new file
+    use file name, not xxx_xxx_status.
+
+    I have FillBlockParam fbp, how to get filepath?
+    fbpFilepath fbp
+  - I don't want to write debugl, infol etc for both client and server side.
+
+    debugl :: RDClientRuntimeConfig -> T.Text -> IO ()
+    infol :: RDClientRuntimeConfig -> T.Text -> IO ()
+    warnl :: RDClientRuntimeConfig -> T.Text -> IO ()
+    errorl :: RDClientRuntimeConfig -> T.Text -> IO ()
+
+    logl :: ToLogStr a => RDRuntimeConfig -> a -> IO ()
+
+    Is this where I need monad transformer?
+
+    loggerSet <- newStdoutLoggerSet defaultBufSize
+    loggerTimeCache <- newTimeCache simpleTimeFormat
+    RDRuntimeConfig { ...
+             , rcLoggerSet=loggerSet
+             , rcLoggerTimeCache=loggerTimeCache
+    }
+
+    RDClientRuntimeConfig {rdLogger=logger}
+
+    yes. this is where ReaderT Env IO comes handy.
+    see test use-case in ~/projects/reliable-download/logtest/Main.hs
+
+    see also
+    The ReaderT Design Pattern
+    https://www.fpcomplete.com/blog/2017/06/readert-design-pattern/
+
+  - still too complicated for me.
+    I start with this:
+    fillSha1sum :: FillBlockParam -> RDApp [BlockWithChecksum]
+
+    don't know how to change:
+    getRdHandler :: RDRuntimeConfig -> ExceptT T.Text ActionM ()
+    to
+
+    getRdHandler :: ExceptT T.Text ActionM RDApp ()
+    or
+    getRdHandler :: ExceptT T.Text RDApp ActionM ()
+
+    need to be more comfortable with monad transformers. and it's best to
+    start it with new project.
+
+    for rd-api, I will just keep RDRuntimeConfig everywhere.
+    just switch to tinylog for logging.
+
 ** 2022-03-11 try build this project using latest lts.
 updated ./stack.yaml
 resolver: lts-18.27

package.yaml

 name:                reliable-download
-version:             1.1.2.0
+version:             1.1.3.0
 synopsis:            provide reliable download service via HTTP
 description:         reliable-download web application and cli tool
 homepage:            "https://gitlab.emacsos.com/sylecn/reliable-download"
@@ -15,6 +15,7 @@ extra-source-files:
 ghc-options:
   - -Wall
   - -Werror
+  - -O2
   # - -fprof-auto
 
 default-extensions:
@@ -47,30 +48,29 @@ dependencies:
   - transformers
   - optparse-applicative
   - formatting
-  - fast-logger
+  - tinylog
   - errors
 
 library:
-  source-dirs: src
+  source-dirs: lib
 
 executables:
-  maybet:
-    source-dirs:      misc/maybet
-    main:             Main.hs
-    dependencies:
-    - reliable-download
-    ghc-options:
-    - -threaded
   rd-api:
-    source-dirs:      api
-    main:             Main.hs
+    source-dirs:
+    - api
+    - api/lib
+    - cli-version
+    main: Main.hs
     dependencies:
     - reliable-download
     ghc-options:
     - -threaded
+    - -O2
   rd:
-    source-dirs:      client
-    main:             Main.hs
+    source-dirs:
+    - client
+    - cli-version
+    main: Main.hs
     dependencies:
     - reliable-download    # I only need the Type module
     - http-conduit
@@ -79,14 +79,28 @@ executables:
     - retry
     - socket
     - io-thread-pool
-    - tinylog
+    ghc-options:
+    - -threaded
+    - -O2
+  maybet:
+    source-dirs: misc/maybet
+    main: Main.hs
+    dependencies:
+    - reliable-download
+    ghc-options:
+    - -threaded
+  logtest:
+    source-dirs: misc/logtest
+    main: Main.hs
     ghc-options:
     - -threaded
 
 tests:
   all-tests:
     main: Main.hs
-    source-dirs: test
+    source-dirs:
+    - test
+    - api/lib
     dependencies:
     - reliable-download
     - hspec

pypi/rd-api/README.rst

@@ -76,6 +76,11 @@ easy installation on linux system. Reliable download only runs in linux.
 ChangeLog
 ---------
 
+* v1.1.3.0 2022-03-14
+
+  - bugfix: revised logging messages. rd-api supports --verbose option. debug msg is not shown by default.
+  - feature: code ported to ghc 8.10.7
+
 * v1.1.0.0 2018-05-10
 
   - feature: support passing arguments using env variables, for cli arg --redis-host, the env variable will be REDIS_HOST.

pypi/rd-client/README.rst

@@ -51,6 +51,10 @@ see rd-api_.
 ChangeLog
 ---------
 
+* v1.1.3.0 2022-03-14
+
+  - feature: code ported to ghc 8.10.7
+
 * v1.0.0.3 2018-05-09
 
   - init release.