generate block metadata without sha1sum is working

module Main (main) where

import Network.Socket.Internal (PortNumber)

import Data.String (fromString)

import System.Environment (lookupEnv)

import Data.Monoid ((<>))

import qualified Data.Text as T

import qualified Data.Text.Lazy as LT

import Web.Scotty

import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort)

import Formatting

import Log

import Log.Backend.StandardOutput

import qualified Database.Redis as R

import App (mkApp)

import Config

-- TODO use a proper config lib.

-- TODO support other env variables.

updateRDConfigFromEnv :: RDConfig -> IO RDConfig

updateRDConfigFromEnv config = do

  webroot <- lookupEnv "WEB_ROOT"

  let newConfig =

          case webroot of

            Just dir -> config {webRoot=dir}

            Nothing -> config

  withSimpleStdOutLogger $ \logger -> do

    runLogT "Main" logger $ do

      logInfo_ $ "webRoot is " <> T.pack (webRoot config)

  return newConfig

main :: IO ()

main = do

  conn <- R.checkedConnect R.defaultConnectInfo

  putStrLn "listening on 0.0.0.0:8082"

  scotty 8082 $ mkApp conn

main = withSimpleStdOutLogger $ \logger -> do

  config <- updateRDConfigFromEnv defaultRDConfig

  conn <- R.checkedConnect $ R.defaultConnectInfo {

            R.connectHost=redisHost config

          , R.connectPort=R.PortNumber $ (fromIntegral (redisPort config) :: PortNumber)

          }

  let runtimeConfig = RDRuntimeConfig { config=config

                                      , redisConn=conn

                                      }

  runLogT "Main" logger $ do

    logInfo_ $ sformat ("will listen on " % string % ":" % int) (host config) (port config)

  let opts = Options { verbose=0

                     , settings=warpSettings

                     }

          where

            warpSettings = setPort (port config)

                           (setHost (fromString $ host config) defaultSettings)

  scottyOpts opts $ mkApp runtimeConfig

Time-stamp: <2018-05-05>

#+STARTUP: content

* notes                                                               :entry:

** 2018-05-05 it's impossible to do logging easily in haskell.

two problems

- no way to pass around the logger engine/configuration without a log monad in

  your stack. since you may want to log anywhere, you have to change all the

  data types.

  This requires you know monad and monad transformer really well.

  You can't just get a logger from "global env" like in other language.

- no easy way to format a string as Data.Text.Text.

  both printf and Text.Format.format has too much noise when doing logging.

  I need to write:

  runLogT "Main" logger $ do

    logInfo_ $ LT.toStrict $ TF.format "will listen on {}:{}" ((host config), (port config))

  I would like to write:

  logInfo_ $ format "will listen on {}:{}" (host config) (port config)

  try https://hackage.haskell.org/package/formatting

  Formatting in Haskell

  https://chrisdone.com/posts/formatting

  logInfo_ $ sformat ("will listen on " % string % ":" % int) (host config) (port config)

  // this is better.

** 2018-05-05 writing tests in hspec

- Test WAI application using hspec

  hspec/hspec-wai: Helpers to test WAI application with Hspec

- 

* later                                                               :entry:

** 2018-05-05 hosting static file via warp?

- try it. it makes development easier.

- make sure Range header is supported.

- 

** 2018-05-05 add logging support

- hslogger

  http://hackage.haskell.org/package/hslogger-1.2.10/docs/System-Log-Logger.html

  http://hackage.haskell.org/package/hslogger

  doesn't support printf style param?

- How do I do logging in Haskell? - Stack Overflow

  https://stackoverflow.com/questions/6310961/how-do-i-do-logging-in-haskell

  says hslogger is not good.

  slow and not extensible.

- logger

  http://hackage.haskell.org/package/logger-0.1.0.2

  not maintained anymore?

  dependencies requires base <=4.9

  lts-10.3 has base 4.10.1.0

- check pkg list.

  https://www.stackage.org/lts-10.3

- fast-logger 2.4.10

  kazu-yamamoto/logger: A fast logging system for Haskell

  https://github.com/kazu-yamamoto/logger

  tinylog :: Stackage Server

  https://www.stackage.org/lts-10.3/package/tinylog-0.14.0

  Simplistic logging using fast-logger.

- log-base :: Stackage Server

  https://www.stackage.org/package/log-base

  Structured logging solution (base package) https://github.com/scrive/log

- try log-base

  it supports stdout, ES, pg.

  it supports lts-11.x

  it supports log levels.

** 2018-05-05 allow config app at runtime.

via env var and command line parameter.

- HOST

- PORT

- REDIS_HOST

- REDIS_PORT

- WEB_ROOT    web root dir, HTTP Path will be relative to this dir.

- 

** 2018-05-05 utf-8 character not working well in path.

curl http://localhost:8082/rd/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%E5%90%8D.rar

{"ok":true,"path":"中"}

* current                                                             :entry:

** 

** 2018-05-05 calculate sha1sum for blocks using a thread pool.

** 2018-05-05 write the main logic of creating block metadata.

then make it work with a thread pool with a single thread.

env WEB_ROOT=/home/sylecn/persist/cache stack exec rd-api

curl -XGET http://localhost:8082/rd/ideaIC-2018.1.tar.gz

this should return json of the block metadata.

- block metadata looks like this:

  GET /rd/bigfile

  #+BEGIN_SRC sh

    {"ok": true,

     "block_size": "2MiB",    # this is a fixed value.

     "file_size": xxxx,       # file size in bytes

     "block_count": 24,

     "blocks": [

             [0, 0, 2097151, block1_sha1sum],

             [1, 2097152, 4194303, block2_sha1sum],

             ...

             [N, start, end, blockN_sha1sum]

     ]}

  #+END_SRC

- problems

  - it hangs.

    curl -XGET http://localhost:8082/rd/ideaIC-2018.1.tar.gz

    curl -XGET http://localhost:8082/test/rd/ideaIC-2018.1.tar.gz

    fixed.

* done                                                                :entry:

** 2018-05-04 check whether haskell is viable for this project.

In static file hosting nginx reverse proxy /rd/ to rd application server.

dependencies:

  - base >= 4.7 && < 5

  - wai

  - warp

  - scotty

  - cryptohash

  - hedis

  - text

  - byteable

  - aeson

  - network

  - log-base

  - formatting

  - filepath

  - unix

library:

  source-dirs: src

module App (mkApp, mkWaiApp) where

module App (mkApp, mkWaiApp, genBlocks) where

import Control.Monad.IO.Class (liftIO)

import Data.Either (fromRight)

import Data.Monoid ((<>))

import qualified Data.Text as T

import qualified Data.Text.Lazy as LT

import Network.Wai (Application)

import Web.Scotty

import Data.Aeson (Value(..), toJSON, object, (.=))

import Data.Either (fromRight)

import qualified Data.Text.Lazy as LT

import System.FilePath (combine)

import System.Posix.Files (getFileStatus, fileSize)

import qualified Database.Redis as R

import Config

import RD.Lib (sha1sum)

type BlockID = Integer

type Block = (BlockID, Integer, Integer)

type BlockWithChecksum = (BlockID, Integer, Integer, T.Text)

genBlocks :: Integer -> Integer -> [Block]

genBlocks fileSize blockSize = if fileSize == 0 then

                                   []

                               else

                                   go (0 :: Integer) (0 :: Integer) []

  where

    go :: Integer -> Integer -> [Block] -> [Block]

    go blockId startByte accumulator =

        if fileSize - startByte == blockSize then

            reverse ((blockId, startByte, fileSize - 1):accumulator)

        else if fileSize - startByte > blockSize then

            go (blockId + 1)

               (startByte + blockSize)

               ((blockId, startByte, startByte + blockSize - 1):accumulator)

        else if startByte < fileSize then

            reverse ((blockId, startByte, fileSize - 1):accumulator)

        else

            reverse accumulator

-- | given a redis connection pool, return a Scotty app.

mkApp :: R.Connection -> ScottyM ()

mkApp conn = do

mkApp :: RDRuntimeConfig -> ScottyM ()

mkApp runtimeConfig = do

  get (literal "/rd/") $ do

    json $ object [("ok" .= True)

                  ,("app" .= ("reliable-download api" :: String))]

                  ,("app" .= ("reliable-download api" :: T.Text))]

  get (regex "^/rd/(.*)") $ do

    path :: LT.Text <- param "1"

    fullPath :: LT.Text <- param "0"

    let filepath = combine (webRoot (config runtimeConfig)) (LT.unpack path)

    liftIO $ putStrLn $ "get block metadata for " <> filepath

    fileStatus <- liftIO $ getFileStatus filepath    -- TODO catch IO exception

    let fileSizeInByte = toInteger $ fileSize fileStatus

        blockSizeInByte = 2097152    -- 2MiB

        blockCount = (fileSizeInByte - 1) `div` blockSizeInByte + 1

        blocks = genBlocks fileSizeInByte blockSizeInByte

    json $ object [("ok" .= True)

                  ,("block_size" .= ("2MiB" :: T.Text))

                  ,("file_size" .= fileSizeInByte)

                  ,("block_count" .= blockCount)

                  ,("blocks" .= blocks)

                  ,("path" .= path)

                  ,("filepath" .= filepath)

                  ]

  get (regex "^/test/rd/(.*)") $ do  -- for testing path capture

    path :: LT.Text <- param "1"

    let filepath = combine (webRoot (config runtimeConfig)) (LT.unpack path)

    json $ object [("ok" .= True)

                  ,("full_path" .= fullPath)

                  ,("path" .= path)]

                  ,("path" .= path)

                  ,("filepath" .= filepath)

                  ]

  get "/debug/t1" $ do

    sha1 <- liftIO $ sha1sum "/home/sylecn/persist/cache/ideaIC-2018.1.tar.gz"

    json $ object ["ok" .= True

                  ,"sha1sum" .= sha1]

  get "/debug/count" $ do

    count <- liftIO $ R.runRedis conn $ do

    count <- liftIO $ R.runRedis (redisConn runtimeConfig) $ do

                        count <- R.incr "count"

                        return count

    json $ object ["ok" .= True

                  ,"count" .= fromRight 0 count]

-- | given a redis connection pool, return a WAI app.

mkWaiApp :: R.Connection -> IO Application

mkWaiApp conn = scottyApp (mkApp conn)

mkWaiApp :: RDRuntimeConfig -> IO Application

mkWaiApp = scottyApp . mkApp

module Config where

import qualified Database.Redis as R

data RDConfig = RDConfig {

      host :: String

    , port :: Int

    , redisHost :: String

    , redisPort :: Int

    , webRoot :: FilePath

    } deriving (Show)

data RDRuntimeConfig = RDRuntimeConfig {

      config :: RDConfig

    , redisConn :: R.Connection

    }

defaultRDConfig :: RDConfig

defaultRDConfig = RDConfig {

                    host = "0.0.0.0"

                  , port = 8082

                  , redisHost = "127.0.0.1"

                  , redisPort = 6379

                  , webRoot = "/nonexistent"

                  }