diff --git a/country-code-generation/LICENSE b/country-code-generation/LICENSE deleted file mode 100644 index 6cc70c5..0000000 --- a/country-code-generation/LICENSE +++ /dev/null @@ -1,30 +0,0 @@ -Copyright Andrew Martin (c) 2017 - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of Andrew Martin nor the names of other - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/country-code-generation/README.md b/country-code-generation/README.md deleted file mode 100644 index 281a219..0000000 --- a/country-code-generation/README.md +++ /dev/null @@ -1 +0,0 @@ -# country diff --git a/country-code-generation/app/Main.hs b/country-code-generation/app/Main.hs deleted file mode 100644 index e3b7d16..0000000 --- a/country-code-generation/app/Main.hs +++ /dev/null @@ -1,317 +0,0 @@ -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE OverloadedStrings #-} - -import Streaming (Stream,Of(..)) -import Siphon (Siphon) -import Colonnade (Headed) -import Data.ByteString (ByteString) -import Data.Text (Text) -import System.IO -import Data.Text.Encoding (encodeUtf8,decodeUtf8') -import Data.Char (isAlpha,toLower) -import Data.DisjointSet (DisjointSet) -import Control.Monad.Trans.Class -import Data.Foldable (for_) -import Control.Monad -import Control.Monad.IO.Class -import Data.Primitive (readArray,writeArray,newArray) -import qualified Data.Set as Set -import qualified Data.DisjointSet as DS -import qualified Streaming as SM -import qualified Streaming.Prelude as SMP -import qualified Siphon as S -import qualified Data.Text as T -import qualified Data.Text.IO as TIO -import qualified Data.Text.Lazy as LT -import qualified Data.Text.Lazy.Builder as TB -import qualified Data.Text.Lazy.Builder.Int as TBI -import qualified Data.ByteString.Char8 as BC -import qualified Data.ByteString.Streaming as BSM - -main :: IO () -main = do - withCountries "country/src/Country/Unexposed/Encode/English.hs" englishEncoding - withCountries "country/src/Country/Identifier.hs" identifierModule - withCountries "country/src/Country/Unexposed/Alias.hs" aliasModule - withCountries "country/src/Country/Unexposed/AlphaTwoPtr.hs" alphaTwoPtrModule - withCountries "country/src/Country/Unexposed/Continents.hs" continents - -alphaTwoPtrModule :: Stream (Of Country) IO r -> Stream (Of Text) IO r -alphaTwoPtrModule s = do - aliasGroups <- lift buildAliasGroups - SMP.yield "{-# language MagicHash #-}\n" - SMP.yield "-- This module is autogenerated. Do not edit it by hand.\n" - SMP.yield "module Country.Unexposed.AlphaTwoPtr\n" - SMP.yield " ( alphaTwoPtr\n" - SMP.yield " ) where\n\n" - SMP.yield "import GHC.Exts (Ptr(Ptr))\n" - SMP.yield "import Data.Word (Word8)\n" - SMP.yield "\n" - SMP.yield "alphaTwoPtr :: Ptr Word8\n" - SMP.yield "alphaTwoPtr = Ptr \"\\\n" - arr <- liftIO (newArray 2000 'A') - r <- flip mapStreamM s $ \country -> do - let (c1,c2) = countryAlpha2 country - code = countryCode country - if code >= 1000 - then error "Encountered country code over 1000" - else liftIO $ do - writeArray arr (2 * code) c1 - writeArray arr (2 * code + 1) c2 - forM_ (enumFromTo 0 999) $ \ix -> do - c1 <- liftIO $ readArray arr (2 * ix) - c2 <- liftIO $ readArray arr (2 * ix + 1) - SMP.yield (T.singleton '\\') - SMP.yield (T.singleton c1) - SMP.yield (T.singleton c2) - SMP.yield (T.singleton '\\') - SMP.yield (T.singleton '\n') - SMP.yield "\\\"#\n" - pure r - -aliasModule :: Stream (Of Country) IO r -> Stream (Of Text) IO r -aliasModule s = do - aliasGroups <- lift buildAliasGroups - SMP.yield "-- This module is autogenerated. Do not edit it by hand.\n" - SMP.yield "module Country.Unexposed.Alias\n" - SMP.yield " ( aliases\n" - SMP.yield " ) where\n\n" - SMP.yield "import Data.Text (Text)\n" - SMP.yield "import Data.Word (Word16)\n" - SMP.yield "import qualified Data.Text as T\n" - SMP.yield "\n" - SMP.yield "aliases :: [(Word16,Text)]\n" - SMP.yield "aliases =\n" - r <- flip mapStreamM (tagFirst s) $ \(isFirst,country) -> do - let name = countryName country - allNames = DS.equivalences name aliasGroups - if isFirst - then SMP.yield " [ (" - else SMP.yield " , (" - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield ", T.pack \"" - SMP.yield (countryName country) - SMP.yield "\")\n" - for_ allNames $ \altName -> when (altName /= name) $ do - SMP.yield " , (" - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield ", T.pack \"" - SMP.yield altName - SMP.yield "\")\n" - SMP.yield " ]\n" - SMP.yield "{-# NOINLINE aliases #-}\n" - return r - -buildAliasGroups :: IO (DisjointSet Text) -buildAliasGroups = do - t <- TIO.readFile "aliases.txt" - let countryGroups = map (filter (not . T.null)) $ (map.map) T.strip $ map T.lines $ T.splitOn "\n\n" t - let res = foldMap (DS.singletons . Set.fromList) countryGroups - return res - -identifierModule :: Stream (Of Country) IO r -> Stream (Of Text) IO r -identifierModule s = do - aliasGroups <- lift buildAliasGroups - SMP.yield "module Country.Identifier where\n\n" - SMP.yield "-- This module is autogenerated. Do not edit it by hand.\n\n" - SMP.yield "import Country.Unsafe (Country(..))\n" - SMP.yield "\n" - flip mapStreamM s $ \country -> do - let name = countryName country - identifier = toIdentifier name - allNames = DS.equivalences name aliasGroups - SMP.yield "-- | " - SMP.yield $ T.intercalate "; " (Set.toList allNames) - SMP.yield "\n" - SMP.yield identifier - SMP.yield " :: Country\n" - SMP.yield identifier - SMP.yield " = Country " - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield "\n\n" - SMP.yield "pattern " - SMP.yield patternName - SMP.yield " :: Country\n" - SMP.yield "pattern " - SMP.yield patternName - SMP.yield " = Country " - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield "\n\n" - -toIdentifier :: Text -> Text -toIdentifier t = case (T.uncons . T.filter isAlpha . T.toTitle) t of - Nothing -> T.empty - Just (b,bs) -> T.cons (toLower b) bs - -toPatternName :: Text -> Text -toPatternName = T.filter isAlpha . T.toTitle - -englishEncoding :: Monad m => Stream (Of Country) m r -> Stream (Of Text) m r -englishEncoding s = do - SMP.yield "-- This module is autogenerated. Do not edit it by hand.\n" - SMP.yield "module Country.Unexposed.Encode.English\n" - SMP.yield " ( countryNameQuads\n" - SMP.yield " ) where\n\n" - SMP.yield "import Data.Text (Text)\n" - SMP.yield "import Data.Word (Word16)\n" - SMP.yield "import qualified Data.Text as T\n" - SMP.yield "\n" - SMP.yield "-- first value is country code, second is english name, \n" - SMP.yield "-- third is two char code, fourth is three char code.\n" - SMP.yield "countryNameQuads :: [(Word16,Text,(Char,Char),(Char,Char,Char))]\n" - SMP.yield "countryNameQuads =\n" - r <- flip mapStreamM (tagFirst s) $ \(isFirst,country) -> do - let (a1,a2) = countryAlpha2 country - (b1,b2,b3) = countryAlpha3 country - if isFirst - then SMP.yield " [ (" - else SMP.yield " , (" - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield ", T.pack \"" - SMP.yield (countryName country) - SMP.yield "\",('" - SMP.yield (T.singleton a1) - SMP.yield "','" - SMP.yield (T.singleton a2) - SMP.yield "'),('" - SMP.yield (T.singleton b1) - SMP.yield "','" - SMP.yield (T.singleton b2) - SMP.yield "','" - SMP.yield (T.singleton b3) - SMP.yield "')" - SMP.yield ")\n" - SMP.yield " ]\n" - SMP.yield "{-# NOINLINE countryNameQuads #-}\n" - return r - -continents :: Stream (Of Country) IO r -> Stream (Of Text) IO r -continents s = do - SMP.yield "-- This module is autogenerated. Do not edit it by hand.\n" - SMP.yield "module Country.Unexposed.Continents\n" - SMP.yield " ( continentAList\n" - SMP.yield " ) where\n\n" - SMP.yield "import Continent.Unsafe\n" - SMP.yield "import Data.Word (Word16)\n" - SMP.yield "\n" - SMP.yield "-- first value is country code, second is the continent constructor\n" - SMP.yield "continentAList :: [(Word16,Continent)]\n" - SMP.yield "continentAList =\n" - r <- flip mapStreamM (tagFirst s) $ \(isFirst,country) -> do - if isFirst - then SMP.yield " [ (" - else SMP.yield " , (" - yieldLazyText (TB.toLazyText (TBI.decimal (countryCode country))) - SMP.yield "," - SMP.yield $ case countryRegion country of - "Asia" -> "Asia" - "Europe" -> "Europe" - "Africa" -> "Africa" - "Oceania" -> "Oceania" - "Americas" -> case countrySubregion country of - "Caribbean" -> "NorthAmerica" - "South America" -> "SouthAmerica" - "Northern America" -> "NorthAmerica" - "Central America" -> "NorthAmerica" - it -> error $ "unrecognized region (\"Americas\") and sub-region (" ++ show it ++ ")" - "" -> case countryName country of - "Antarctica" -> "Antarctica" - "Bouvet Island" -> "Antarctica" -- WARNING I'm making this one up - "British Indian Ocean Territory" -> "Oceania" -- WARNING I'm making this one up - "Christmas Island" -> "Oceania" -- WARNING I'm making this one up - "Cocos (Keeling) Islands" -> "Oceania" -- WARNING I'm making this one up - "French Southern Territories" -> "Antarctica" -- WARNING I'm making this one up - "Heard Island and McDonald Islands" -> "Antarctica" -- WARNING I'm making this one up - "South Georgia and the South Sandwich Islands" -> "Antarctica" -- WARNING I'm making this one up - "United States Minor Outlying Islands" -> "Oceania" -- WARNING I'm making this one up - it -> error $ "unrecognized region (\"\") and sub-region (" ++ show it ++ ")" - it -> error $ "unrecognized region (" ++ show it ++ ")" - SMP.yield ")\n" - SMP.yield " ]\n" - SMP.yield "{-# NOINLINE continentAList #-}\n" - return r - -yieldLazyText :: Monad m => LT.Text -> Stream (Of Text) m () -yieldLazyText = mapM_ SMP.yield . LT.toChunks - -tagFirst :: Monad m => Stream (Of a) m r -> Stream (Of (Bool,a)) m r -tagFirst = SMP.zip (SMP.yield True >> SMP.repeat False) - -mapStreamM :: Monad m - => (a -> Stream (Of b) m x) - -> Stream (Of a) m r - -> Stream (Of b) m r -mapStreamM f = SM.concats . SM.mapsM (\(a :> s) -> return (f a >> return s)) - -withCountries :: - String -- ^ file name - -> (forall r. Stream (Of Country) IO r -> Stream (Of Text) IO r) - -> IO () -withCountries fn g = - withFile fn WriteMode $ \output -> - withFile "countries.csv" ReadMode $ \input -> do - m <- id - $ BSM.hPut output - $ BSM.fromChunks - $ SMP.map encodeUtf8 - $ g - $ S.decodeCsvUtf8 siphon - $ BSM.toChunks - $ BSM.fromHandle input - case m of - Nothing -> return () - Just err -> do - hPutStrLn stderr (S.humanizeSiphonError err) - fail "died" - -data Country = Country - { countryName :: Text - , countryAlpha2 :: (Char,Char) - , countryAlpha3 :: (Char,Char,Char) - , countryCode :: Int - , countryRegion :: Text - , countrySubregion :: Text - } - -siphon :: Siphon Headed ByteString Country -siphon = Country - <$> S.headed "name" decodeUtf8Maybe - <*> S.headed "alpha-2" decodeChar2 - <*> S.headed "alpha-3" decodeChar3 - <*> S.headed "country-code" decodeInt - <*> S.headed "region" decodeUtf8Maybe - <*> S.headed "sub-region" decodeUtf8Maybe - -decodeUtf8Maybe :: ByteString -> Maybe Text -decodeUtf8Maybe = either (\_ -> Nothing) Just . decodeUtf8' - -decodeChar2 :: ByteString -> Maybe (Char,Char) -decodeChar2 bs = if BC.length bs == 2 - then - let b0 = BC.index bs 0 - b1 = BC.index bs 1 - in if isUpperAscii b0 && isUpperAscii b1 - then Just (b0,b1) - else Nothing - else Nothing - -isUpperAscii :: Char -> Bool -isUpperAscii x = x >= 'A' && x <= 'Z' - -decodeChar3 :: ByteString -> Maybe (Char,Char,Char) -decodeChar3 bs = if BC.length bs == 3 - then - let b0 = BC.index bs 0 - b1 = BC.index bs 1 - b2 = BC.index bs 2 - in if isUpperAscii b0 && isUpperAscii b1 && isUpperAscii b2 - then Just (b0,b1,b2) - else Nothing - else Nothing - -decodeInt :: ByteString -> Maybe Int -decodeInt b = do - (a,bsRem) <- BC.readInt b - if BC.null bsRem - then Just a - else Nothing diff --git a/country-code-generation/country-code-generation.cabal b/country-code-generation/country-code-generation.cabal deleted file mode 100644 index 902b8ac..0000000 --- a/country-code-generation/country-code-generation.cabal +++ /dev/null @@ -1,38 +0,0 @@ -cabal-version: 3.0 -name: country-code-generation -version: 0.1.0.0 -homepage: https://github.com/byteverse/country -bug-reports: https://github.com/byteverse/country/issues -license: BSD-3-Clause -license-file: LICENSE -author: Andrew Martin -maintainer: amartin@layer3com.com -copyright: 2017 Andrew Martin -category: Web -build-type: Simple - -common build-settings - default-language: Haskell2010 - ghc-options: -Wall -Wunused-packages - -executable country-code-generation - import: build-settings - ghc-options: -O2 - hs-source-dirs: app - main-is: Main.hs - build-depends: - , base <5.0 - , bytestring >=0.10 - , colonnade >=1.2.0.1 - , containers - , disjoint-containers >=0.3 - , primitive - , siphon >=0.8.1 - , streaming - , streaming-bytestring - , text >=1.2 && <2.1 - , transformers - -source-repository head - type: git - location: git://github.com/byteverse/country.git