redis works with haskell

module Main (main) where

import Web.Scotty

import qualified Database.Redis as R

import App (app)

import App (mkApp)

main :: IO ()

main = do

  conn <- R.checkedConnect R.defaultConnectInfo

  putStrLn "listening on 0.0.0.0:8082"

  scotty 8082 app

  scotty 8082 $ mkApp conn

- 

* later                                                               :entry:

** 2018-05-05 should rd-api serve the static file itself?

that way you don't need to tell rd-server the web root dir.

- normally

  nginx serve static files under $WEB_ROOT

  rd-server --web-root $WEB_ROOT

  this serve /rd/.*, provides metadata for files.

- it may be faster if you just do

  rd-server --web-root $WEB_ROOT

  This serves both /rd/.* and /.*

* current                                                             :entry:

** 

** 2018-05-05 utf-8 character not working well.

** 2018-05-05 utf-8 character not working well in path.

curl http://localhost:8082/rd/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%E5%90%8D.rar

{"ok":true,"path":"中"}

only first character is in path key.

- search: haskell scotty path utf-8 character

  check source code.

  Web.Scotty.Route

  https://www.stackage.org/haddock/lts-11.7/scotty-0.11.1/src/Web.Scotty.Route.html

  matchRoute

  path req

  req :: Request

  path :: Request -> T.Text

  path = T.fromStrict . TS.cons '/' . TS.intercalate "/" . pathInfo

  pathInfo is defined in Network.Wai!

  so problem is not in scotty. probably in Warp.

- check Network.Wai.pathInfo source code

  https://www.stackage.org/haddock/lts-11.7/wai-3.2.1.2/Network-Wai.html#v:pathInfo

  data Request = Request {

      ,  pathInfo             :: [Text]

  }

  check how warp fill this field.

  Network.Wai.Handler.Warp.Request

  https://www.stackage.org/haddock/lts-11.7/warp-3.2.22/src/Network.Wai.Handler.Warp.Request.html

  import qualified Network.HTTP.Types as H

  hdrlines <- headerLines firstRequest src

  (method, unparsedPath, path, query, httpversion, hdr) <- parseHeaderLines hdrlines

  pathInfo = H.decodePathSegments path

  H.decodePathSegments doesn't have problem. I already tested it.

- check parseHeaderLines

  https://www.stackage.org/haddock/lts-11.7/warp-3.2.22/src/Network.Wai.Handler.Warp.RequestHeader.html#parseHeaderLines

  try parse this line using warp's code:

  GET /rd/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%E5%90%8D.rar HTTP/1.1

  stack repl

  import Network.Wai.Handler.Warp.RequestHeader (parseHeaderLines)

  :l ~/fromsource/wai/warp/Network/Wai/Handler/Warp/RequestHeader

  too many dependencies

  try build warp in it's source dir.

  cd ~/fromsource/wai/warp

  stack build

  lts-10.0 plan.

  96 pkgs to build.

- problems

  - can't find warp source code.

    warp is inside wai repo.

    https://github.com/yesodweb/wai

    cloned to ~/fromsource/wai/

    just import Network.Wai.Handler.Warp.Internal

    it includes every module. but it doesn't export that function.

  - search: haskell how to use non exported function

** 2018-05-05 should rd-api serve the static file itself?

that way you don't need to tell rd-server the web root dir.

- normally

  nginx serve static files under $WEB_ROOT

  rd-server --web-root $WEB_ROOT

  this serve /rd/.*, provides metadata for files.

- it may be faster if you just do

  rd-server --web-root $WEB_ROOT

  This serves both /rd/.* and /.*

* current                                                             :entry:

** 

* done                                                                :entry:

** 2018-05-04 check whether haskell is viable for this project.

In static file hosting nginx reverse proxy /rd/ to rd application server.

  - GET /rd/bigfile

  - GET /rd/path/to/bigfile

- redis client

- DONE redis client

- DONE calculate sha1sum in consistent RAM.

  curl http://localhost:8082/

  just scotty 6.6MiB

  this calculate sha1sum for a 517MiB file.

  11.2MiB

  it's okay. doesn't buffer the whole file in RAM.

- 

- haskell is viable for this project.

* done                                                                :entry:

** 2018-05-05 redis client

- when to create the connection pool?

  in main

** 2018-05-05 how to test scotty web api

wai-test: Unit test framework (built on HUnit) for WAI applications. (deprecated)

http://hackage.haskell.org/package/wai-test

  - wai

  - scotty

  - cryptohash

  - hedis

  - bytestring

  - text

  - byteable

module App (app, waiApp) where

module App (mkApp, mkWaiApp) where

import Control.Monad.IO.Class (liftIO)

import Network.Wai (Application)

import Web.Scotty

import Data.Aeson (Value(..), toJSON, object, (.=))

import Data.Either (fromRight)

import qualified Data.Text.Lazy as LT

import qualified Database.Redis as R

import RD.Lib (sha1sum)

app :: ScottyM ()

app = do

-- | given a redis connection pool, return a Scotty app.

mkApp :: R.Connection -> ScottyM ()

mkApp conn = do

  get (literal "/rd/") $ do

    json $ object [("ok" .= True)

                  ,("app" .= ("reliable-download api" :: String))]

    json $ object [("ok" .= True)

                  ,("full_path" .= fullPath)

                  ,("path" .= path)]

  get "/rd/debug/t1" $ do

  get "/debug/t1" $ do

    sha1 <- liftIO $ sha1sum "/home/sylecn/persist/cache/ideaIC-2018.1.tar.gz"

    json $ object ["ok" .= True

                  ,"sha1sum" .= sha1]

  get "/debug/count" $ do

    count <- liftIO $ R.runRedis conn $ do

                        count <- R.incr "count"

                        return count

    json $ object ["ok" .= True

                  ,"count" .= fromRight 0 count]

waiApp :: IO Application

waiApp = scottyApp app

-- | given a redis connection pool, return a WAI app.

mkWaiApp :: R.Connection -> IO Application

mkWaiApp conn = scottyApp (mkApp conn)

import Network.Wai.Test

import Network.HTTP.Types (status200, encodePathSegments, decodePathSegments)

import Data.Binary.Builder (toLazyByteString)

import Network.Wai (Application)

import qualified Data.Aeson as J

import qualified Data.ByteString.Lazy as LB

import qualified Data.Text as T

import qualified Data.HashMap.Strict as H

import qualified Database.Redis as R

import App (waiApp)

import App (mkWaiApp)

-- | decode response as json object.

jsonObject :: SResponse -> Maybe J.Object

getPath :: [T.Text] -> WaiSession SResponse

getPath = get . LB.toStrict . toLazyByteString . encodePathSegments

waiApp :: IO Application

waiApp = do

   conn <- R.connect R.defaultConnectInfo

   mkWaiApp conn

apiSpec :: Spec

apiSpec = with waiApp $ do

  describe "rd api" $ do