update log ts and some msg

pypi/rd-api/rdapi/rd-api

*README.rst

stack.yaml.lock

*.whl

          removeFile targetFilename

          return True)

        (\e -> do

           errorl rc $ "remove existing file failed: " <> showt e

           errorl rc $ "Remove existing file failed: " <> showt e

           return False)

      unless result mzero

  liftIO $ do

    infol rc $ "combining blocks to create " <> showt targetFilename

    infol rc $ "Combining blocks to create " <> showt targetFilename

    forM_ (getBlockTargetFilenames opts rdResp) $ \blockFilename -> do

      debugl rc $ "appending block file " <> showt blockFilename

      content <- LB.readFile blockFilename

      LB.appendFile targetFilename content  -- TODO how to handle error here?

                                            -- let it crash?

    infol rc $ "file downloaded to " <> showt targetFilename

    infol rc $ "File downloaded to " <> showt targetFilename

    unless (keepBlockData opts) $ do

      let tempdir = tempDir opts </> filename

      debugl rc $ "delete temporary block data dir " <> showt tempdir

      debugl rc $ "Delete temporary block data dir " <> showt tempdir

      catchIOError (removeDirectoryRecursive tempdir)

                   (\e -> warnl rc $ "Warning: delete temp block data dir failed: " <> showt e)

cliApp opts = do

  let level = if verbose opts then L.Debug else L.Info

      -- Note: tinylog doesn't support non-GMT dateformat.

      logSettings = (L.setFormat (Just "%0H:%0M:%0S") .

      logSettings = (L.setFormat (Just "%Y-%0m-%0dT%0H:%0M:%0S") .

                     L.setLogLevel level .

                     L.setDelimiter "  ")

                    L.defSettings

  resultsMaybe <- mapM (runMaybeT . downloadFile rc) (urls opts)

  let results = map (fromMaybe False) resultsMaybe

  if and results then

      infol rc "all urls downloaded."

      infol rc "All urls downloaded."

  else do

      errorl rc $ (showt . length . filter not) results <> " urls failed/skipped."

      exitFailure

* COMMENT -*- mode: org -*-

#+Date: 2018-05-04

Time-stamp: <2022-03-12>

Time-stamp: <2022-03-13>

#+STARTUP: content

* notes                                                               :entry:

** 2018-05-09 how to release latest code on PyPI?

  just enough for the user to understand what's going on.

- on client side, "block xx fetched" is not as useful as "xx/xx block fetched,

  xx%"

- implementation

  ./src/App.hs

  mkApp

  getRdHandler

- DONE client side log should have datatime as ts, not just time.

  use the default ISO8601 UTC time.

  it doesn't support local time.

  client side use tinylog

  import qualified System.Logger as L

  and a few wrappers, such as debugl, infol, warnl etc.

  rdLogger is saved in RDClientRuntimeConfig.

  tinylog last release is from 2019.5 no updates since then.

  other logging systems

  katip: A structured logging framework.

  https://hackage.haskell.org/package/katip

  too many dependencies for a logging library.

  simple-logger: A very simple but efficient logging framework

  https://hackage.haskell.org/package/simple-logger

  last release 2020.12

  heavy-logger: Full-weight logging based on fast-logger

  https://hackage.haskell.org/package/heavy-logger

  co-log :: Kowainik

  https://kowainik.github.io/projects/co-log

  co-log/tutorials at main · co-log/co-log

  https://github.com/co-log/co-log/tree/main/tutorials

  still too complicated because I don't use monad transformers in my app.

  How do I do logging in Haskell? - Stack Overflow

  https://stackoverflow.com/questions/6310961/how-do-i-do-logging-in-haskell

  just continue to use tinylog.

- ~/projects/reliable-download/src/Utils.hs

  rd-api use fast-logger

  import System.Log.FastLogger

  fast-logger: A fast logging system

  https://hackage.haskell.org/package/fast-logger-3.1.1

  last release is 2022.1, good.

  -- | log a message that implements ToLogStr

  logl :: ToLogStr a => RDRuntimeConfig -> a -> IO ()

  I can implement log level here myself.

  logl rc $

  try convert from fast-logger to tinylog.

  api/Main.hs

  src/App.hs

  src/DB.hs

  src/Utils.hs    drop logl definition.

  src/Worker.hs

- problems

  - rd client has log level. api doesn't have log level.

    I need to add log level to api.

    fillSha1sum: redis hgetall "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152" ok

    redis hset "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152" 0 ok

    set file status to "done" for "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl"

    these should be debug log.

  - DONE 12/Mar/2022:23:01:15 +0800 "/home/sylecn/d/t2/wheelhouse/cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl_2097152_status" is not a new file

    use file name, not xxx_xxx_status.

    I have FillBlockParam fbp, how to get filepath?

    fbpFilepath fbp

  - 

** 2021-10-07 When combine blocks to final file, do not use more disk space than original file size.

processNewFileAsyncMaybe :: RDRuntimeConfig -> FillBlockParam -> ExceptT T.Text IO ()

processNewFileAsyncMaybe rc fbp = do

  let strKey = fileStatusKey fbp

      filePath = fbpFilepath fbp

  resultE <- liftIO $ DB.insertIfNotExist rc strKey fileStatusWorking

  throwOnLeft resultE

  let insertOk = fromRight' resultE

  if insertOk then liftIO $ do

      logl rc $ showt strKey <> " is a new file, sending task to worker"

      logl rc $ showt filePath <> " is a new file, sending task to worker"

      writeChan (rcFileChan rc) fbp

      return ()

  else do

    oldStatusE <- liftIO $ do

      logl rc $ showt strKey <> " is not a new file"

      logl rc $ showt filePath <> " is not a new file"

      -- if status is error, set it to working, then add task to fileChan

      DB.get rc strKey

    throwOnLeftMsg oldStatusE "get old file status failed"

module CliVersion where

cliVersion :: String

cliVersion = "1.1.0.0"

cliVersion = "1.1.2.0"