Commit 8e6ca421 authored by Yuanle Song's avatar Yuanle Song
Browse files

v1.3.2.0 give fileWorker a name, so logs are more accurate

parent 4f382c07

operational

+22 −19
Original line number Diff line number Diff line
@@ -494,25 +494,6 @@ via env var and command line parameter. 


* current                                                             :entry:

** 

** 2024-04-26 log refine.

- DONE 2024-04-26T19:06:17  I  fileWorker done for /home/sylecn/d/t2/foo, 0.0 MiB, 1 blocks

  don't use 0.0MiB, just say <1MiB or so.



  humanReadableSize



  add unit test for this.



- 2024-04-26T19:12:02  I  fileWorker is waiting for jobs...

  2024-04-26T19:12:02  I  fileWorker is waiting for jobs...



  fileWorker-<N> is waiting for jobs



  2024-04-26T19:12:02  I  fileWorker working on /home/sylecn/d/t2/foo

  2024-04-26T19:12:02  I  fileWorker done for /home/sylecn/d/t2/foo, 0.0 MiB, 1 blocks



  fileWorker-<N> working on ...

- 



** 2024-03-12 rd-api, if file is already transferred block by block, I can support

live compress easily. If the client request compress as param such as

?compress=zstd. default is no compress.
@@ -541,6 +522,28 @@ policy in ovs can do it. 
  see stretch01 daylog.



* done                                                                :entry:

** 2024-04-26 log refine.

- DONE 2024-04-26T19:06:17  I  fileWorker done for /home/sylecn/d/t2/foo, 0.0 MiB, 1 blocks

  don't use 0.0MiB, just say <1MiB or so.



  humanReadableSize



  add unit test for this.



- DONE 2024-04-26T19:12:02  I  fileWorker is waiting for jobs...

  2024-04-26T19:12:02  I  fileWorker is waiting for jobs...



  fileWorker-<N> is waiting for jobs



  2024-04-26T19:12:02  I  fileWorker working on /home/sylecn/d/t2/foo

  2024-04-26T19:12:02  I  fileWorker done for /home/sylecn/d/t2/foo, 0.0 MiB, 1 blocks



  fileWorker-<N> working on ...



  in startWorkers, give it a name.



- 



** 2024-04-07 rd-api: when a file content is changed on disk, auto invalidate all

cached blocks.

package.yaml

+1 −1
Original line number Diff line number Diff line 
name:                reliable-download

# do not modify version if lib/ code is not changed.

# real app version: 1.3.1.0

# real app version: 1.3.2.0

version:             1.2.6.0

synopsis:            provide reliable download service via HTTP

description:         reliable-download web application and cli tool

rd-api/RD/Server/Worker.hs

+12 −8
Original line number Diff line number Diff line
@@ -3,7 +3,9 @@ module RD.Server.Worker (startWorkers, sha1sumFileRange, fileRange) where 
import Control.Concurrent.Chan

import System.IO (IOMode(ReadMode), withBinaryFile)

import GHC.IO.Handle (Handle, hTell, hSeek, SeekMode(AbsoluteSeek))

import Control.Monad (when, replicateM_, forever)

import Control.Monad (when, forever)

import Data.Foldable (forM_)



import Control.Concurrent (forkIO)

import qualified Data.ByteString as B

import qualified Data.ByteString.Lazy as LB
@@ -38,14 +40,14 @@ sha1sumFileRange filepath start end = sha1sumOnBytes <$> fileRange filepath star 


-- | a file worker fetch FillBlockParam from fileChan, then calculate sha1sum

-- for all blocks and write result to redis. then mark the file as done.

fileWorker :: RDRuntimeConfig -> IO ()

fileWorker rc = forever $ do

  infol rc ("fileWorker is waiting for jobs..." :: T.Text)

fileWorker :: RDRuntimeConfig -> T.Text -> IO ()

fileWorker rc fileWorkerName = forever $ do

  infol rc $ fileWorkerName <> " is waiting for jobs..."

  fbp <- readChan (rcFileChan rc)

  let filepath = fbpFilepath fbp

      conn = rcRedisConn rc

  -- calculate sha1sum for each block and write result to redis hash

  infol rc $ "fileWorker working on " <> T.pack filepath

  infol rc $ fileWorkerName <> " working on " <> T.pack filepath

  results <- withBinaryFile filepath ReadMode $ \handle -> do

    let hashKey = blockSha1sumHashKey fbp

    mapM (calculateSha1ForBlock conn hashKey handle) (fbpBlocks fbp)
@@ -57,8 +59,9 @@ fileWorker rc = forever $ do 
    Right _ -> do

      debugl rc $ "Set file status to " <> showt resultStatus <> " for " <> T.pack filepath

      infol rc $ sformat

        ("fileWorker done for " % stext % ", " % stext % ", " % int % " blocks")

        (T.pack filepath) (humanReadableSize (fbpFileSize fbp)) (length (fbpBlocks fbp))

        (stext % " done for " % stext % ", " % stext % ", " % int % " blocks")

        fileWorkerName (T.pack filepath) (humanReadableSize (fbpFileSize fbp))

        (length (fbpBlocks fbp))

  return ()

    where

      -- | calculate sha1 for a single block. return IO True on success.
@@ -94,4 +97,5 @@ startWorkers :: RDRuntimeConfig -> IO () 
startWorkers rc = do

  let workerCount = fileWorkerCount $ rcConfig rc

  infol rc $ "creating " <> showt workerCount <> " file worker(s)"

  replicateM_ workerCount (forkIO $ fileWorker rc)
 
  forM_ [1..workerCount]

        (\n -> forkIO $ fileWorker rc ("fileWorker-" <> showt n))