miso is a dependency, locked like the others: not on every project's path

README.md

@@ -180,8 +180,20 @@ first error, so after fixing one, the next shows on the next save.
 
 Some packages come with the tool, in its package database
 (`~/.miso/mhs-VERSION`): the compiler's (`base`, `ghc-compat`, `array-mhs`,
-`template-haskell`), and the libraries `containers`, `mtl`, `transformers`
-and `miso`.  The packages a project installs go in the project, in
+`template-haskell`), and the libraries `containers`, `mtl` and
+`transformers`.
+
+miso is not one of them: a project depends on it (`miso new` writes
+`dependencies: [miso]`), and locks it like any other package.  `- miso`, with
+no source of its own, is the miso of the tool's release: the commit of
+dmjio/miso it was built from (`~/.miso/mhs-VERSION/miso-default`), locked as
+a git dependency at that commit, and installed as it came with the release
+(`~/.miso/mhs-VERSION/miso-prebuilt`), not built.  A new release of the tool
+does not change the miso of a project that has it in `miso.lock`; `miso
+update` moves it to the new release's.  Another miso is a dependency with
+its own source (`git:`, `path:`, or a `version:` from Hackage).  A project
+whose code imports `Miso` but does not depend on it (from before) gets the
+release's, with a warning.  The packages a project installs go in the project, in
 `.miso/` (next to `miso.yml`; `miso new` puts it in `.gitignore`, and
 `miso clean` removes it).  mhs looks for packages in the project's
 `.miso/` first, then in the tool's database, and nowhere else: what one
@@ -209,10 +221,18 @@ it; `miso list` shows which:
 
 ```yaml
 dependencies:
-  - name: transformers     # instead of the tool's 0.6.1.2: mtl and miso,
-    version: 0.6.1.1       # which need it, are built again against it
-  - name: miso             # miso from a checkout (with the flags the tool's
-    path: ../miso          # is built with, in miso-sources/miso-*.flags)
+  - miso                   # built again against transformers 0.6.1.1
+  - name: transformers     # instead of the tool's 0.6.1.2: mtl, which needs
+    version: 0.6.1.1       # it, is built again against it
+```
+
+miso from a checkout is a `path:` dependency, with the flags the release
+builds it with (in `~/.miso/mhs-VERSION/miso-default`):
+
+```yaml
+dependencies:
+  - name: miso
+    path: ../miso
     flags: -template-haskell -aeson -native -production -ssr -benchmark -text
 ```
 
@@ -235,7 +255,7 @@ version of the package there (from its `miso.yml`, or its `.cabal` file):
 # Local (path:) dependencies are not locked.
 miso: 0.1.0
 mhs: 0.16.10.0
-bundled: containers-0.8 miso-1.14.0.0 mtl-2.3.1 transformers-0.6.1.2
+bundled: containers-0.8 mtl-2.3.1 transformers-0.6.1.2
 packages:
   - name: split
     hackage: 0.2.5.1

flake.nix

@@ -48,8 +48,6 @@
             cp build/miso build/cpphs $out/bin/
             cp -r ${pkgs.miso-mhs}/lib/mcabal/mhs-${mhsVersion} $out/mhs-${mhsVersion}
             chmod -R u+w $out/mhs-${mhsVersion}
-            # what comes with the tool (miso list tells these from what users install)
-            (cd $out/mhs-${mhsVersion}/packages && ls *.pkg) > $out/mhs-${mhsVersion}/miso-bundled
             # the sources of its libraries, as miso-mhs built them, for
             # projects that replace one of them (scripts/ship-source.sh)
             db=$out/mhs-${mhsVersion}
@@ -60,6 +58,11 @@
             sh scripts/ship-source.sh $db mtl mtl
             sh scripts/ship-source.sh $db containers ${misoSrc.mhs-containers}/containers
             sh scripts/ship-source.sh $db miso ${misoSrc.miso} "-template-haskell -aeson -native -production -ssr -benchmark -text"
+            # miso is not on every project's package path: a project depends
+            # on it, at the commit of flake.lock, and installs this prebuilt one
+            sh scripts/prebuilt-miso.sh $db https://github.com/dmjio/miso ${miso.rev} "-template-haskell -aeson -native -production -ssr -benchmark -text"
+            # what comes with the tool (miso list tells these from what users install)
+            (cd $db/packages && ls *.pkg) > $db/miso-bundled
             runHook postInstall
           '';
         };

scripts/build-release.sh

@@ -178,6 +178,9 @@ scripts/ship-source.sh "$db" transformers "$work/src/mhs-transformers"
 scripts/ship-source.sh "$db" mtl "$work/src/mhs-mtl"
 scripts/ship-source.sh "$db" containers "$work/src/mhs-containers/containers"
 scripts/ship-source.sh "$db" miso "$work/src/miso" "$misoflags"
+# miso is not on every project's package path: a project depends on it, at
+# the commit of flake.lock, and installs this prebuilt one
+scripts/prebuilt-miso.sh "$db" https://github.com/dmjio/miso "$misorev" "$misoflags"
 
 # 3. The miso tool
 say "building miso"

scripts/prebuilt-miso.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+# Take miso out of the tool's package database, where every project would
+# use it, and keep it next to it, prebuilt: a project depends on miso (in
+# its miso.yml) and locks it like any other package, at a commit of
+# dmjio/miso, and miso install copies this one when it is the one locked.
+#
+#   mhs-VERSION/miso-default    the miso of this release: its repository,
+#                               commit, version, and cabal flags
+#   mhs-VERSION/miso-prebuilt/  the package, as built from them (its .pkg,
+#                               data directory, and module files)
+#
+# scripts/build-release.sh and flake.nix run it, after installing miso.
+# Usage: scripts/prebuilt-miso.sh DB REPOSITORY COMMIT FLAGS
+set -eu
+db=$1 repo=$2 commit=$3 flags=$4
+pkg=$(cd "$db/packages" && ls | grep -E '^miso-[0-9.]+\.pkg$')
+[ -n "$pkg" ] || { echo "prebuilt-miso: no miso package in $db/packages" >&2; exit 1; }
+nv=${pkg%.pkg}
+version=${nv#miso-}
+out=$db/miso-prebuilt
+rm -rf "$out"
+mkdir -p "$out/packages"
+# the module files that name the package (A/B.txt for module A.B)
+(cd "$db" && grep -rlxF --include='*.txt' --exclude-dir=packages "$pkg" .) > "$db/.miso-modules"
+(cd "$db" && tar -cf - -T .miso-modules) | (cd "$out" && tar -xf -)
+(cd "$db" && xargs rm -f < .miso-modules)
+# and the directories of those modules that are empty now
+find "$db" -depth -type d -empty ! -path "$db/packages*" ! -path "$db/miso-prebuilt*" -exec rmdir {} \;
+rm -f "$db/.miso-modules"
+mv "$db/packages/$pkg" "$out/packages/"
+if [ -d "$db/packages/$nv" ]; then mv "$db/packages/$nv" "$out/packages/"; fi
+chmod -R u+w "$out"
+printf 'repository: %s\ncommit: %s\nversion: %s\nflags: %s\n' "$repo" "$commit" "$version" "$flags" > "$db/miso-default"

scripts/smoke-test.sh

@@ -11,7 +11,7 @@ miso new hello
 cd hello
 # data-default-class needs data-default, which does not come with miso:
 # miso installs it too (by name, then locked, then from vendor/)
-sed -e 's/^target: browser/target: native/' -e 's/^dependencies: \[\]/dependencies: [split, data-default-class]/' miso.yml > miso.yml.new
+sed -e 's/^target: browser/target: native/' -e 's/^dependencies: \[miso\]/dependencies: [split, data-default-class]/' miso.yml > miso.yml.new
 mv miso.yml.new miso.yml
 cat > app/Main.hs <<'HS'
 {-# LANGUAGE CPP #-}
@@ -114,7 +114,7 @@ case "$page" in *served*) ;; *) echo "smoke test: miso serve did not serve the p
 cd "$tmp"
 miso new hello2 > /dev/null
 cd hello2
-sed -e 's/^target: browser/target: native/' -e 's/^dependencies: \[\]/dependencies: [split]/' miso.yml > miso.yml.new
+sed -e 's/^target: browser/target: native/' -e 's/^dependencies: \[miso\]/dependencies: [split]/' miso.yml > miso.yml.new
 mv miso.yml.new miso.yml
 printf 'import Data.List.Split (splitOn)\nmain :: IO ()\nmain = putStrLn (concat (splitOn "," "hel,lo"))\n' > app/Main.hs
 miso run > store.log 2>&1 || { echo "smoke test: miso run failed in a second project" >&2; cat store.log >&2; exit 1; }
@@ -146,4 +146,15 @@ sed -e 's/name: transformers/name: base/' -e 's/version: 0.6.1.1/version: 1.0/'
 mv miso.yml.new miso.yml
 if miso install > base.log 2>&1; then echo "smoke test: replacing base did not fail" >&2; exit 1; fi
 grep -q 'base comes with miso as part of its compiler' base.log || { echo "smoke test: no error for replacing base" >&2; cat base.log >&2; exit 1; }
+# a new project depends on miso, the miso of the release: installed as it
+# came with the tool (not built), and locked at its commit
+cd "$tmp"
+miso new hello4 > /dev/null
+cd hello4
+grep -q '^dependencies: \[miso\]' miso.yml || { echo "smoke test: a new project does not depend on miso" >&2; cat miso.yml >&2; exit 1; }
+miso install > miso.log 2>&1 || { echo "smoke test: miso install failed in a new project" >&2; cat miso.log >&2; exit 1; }
+grep -q 'Installing miso .*(built already, ' miso.log || { echo "smoke test: miso was built, not installed as it came" >&2; cat miso.log >&2; exit 1; }
+ls .miso/mcabal/mhs-*/packages/miso-*.pkg >/dev/null || { echo "smoke test: miso is not in .miso/" >&2; exit 1; }
+miso lock > /dev/null
+grep -q '^    git: https://github.com/dmjio/miso$' miso.lock || { echo "smoke test: miso lock did not lock miso" >&2; cat miso.lock >&2; exit 1; }
 echo "smoke test: ok"

src/Miso/CLI/Config.hs

@@ -46,7 +46,9 @@
 -- >     source-dirs: [test]    # default [test]
 -- >     dependencies: [split]
 --
--- miso itself, and base, containers, mtl, ... come installed with the tool.
+-- base, containers, mtl, ... come installed with the tool.  miso is a
+-- dependency like the others: - miso is the miso of the tool's release,
+-- locked at its commit (Miso.CLI.Store).
 -- A dependency on one of those with a source (git:, path:, or a version:
 -- that is not the tool's) is installed in the project instead, and the
 -- tool's packages that need it are built again against it (Miso.CLI.Deps).

src/Miso/CLI/Deps.hs

@@ -82,7 +82,8 @@ data Unit = Unit
 -- a local directory, nor something built against one) is kept in the store
 -- (Miso.CLI.Store), and copied from there when another project needs it.
 installDeps :: Config -> IO ()
-installDeps cfg = do
+installDeps cfg0 = do
+  cfg <- withMiso cfg0
   setupEnv
   root <- projectRoot >>= maybe (failWith ("no " ++ configFile ++ " here")) pure
   let db = projectDb root
@@ -231,22 +232,31 @@ installLocked _ db _ tool replaced items _ = do
   done0 <- readStamp db
   let done = [ k | k <- done0, not (any (\n -> (n ++ "-") `isPrefixOf` k) dropped) ]
   -- (a local package is installed again when its files change)
-  let todo = [ (u, k) | ((u, _), (_, k)) <- zip ordered ids, keyStamp k `notElem` done ]
+  let todo = [ (u, ds, k) | ((u, ds), (_, k)) <- zip ordered ids, keyStamp k `notElem` done ]
+      planned = map (nameOf . fst) ordered
   if null todo
     then step "dependencies are installed"
-    else forM_ todo $ \(u, k) -> do
+    else forM_ todo $ \(u, ds, k) -> do
       let d = uDep u
           ver = case (uLocked u, uTool u) of
                   (Just l, _) -> lockedAt l
                   (_, Just b) -> " " ++ bVersion b
                   _ -> maybe "" (" " ++) (depVersion d)
           what = depName d ++ ver
       fromCache <- case k of
-        Stored sid _ -> do
+        Stored sid keyLines -> do
           maybe (pure ()) (\v -> removeOtherVersions db (nameOf u) v) (unitVersion u)
           got <- fromStore db sid
           when got $ info ("Installing " ++ what ++ " (built already, from the store)")
-          pure got
+          -- the release's miso, built against the tool's packages
+          pre <- case uLocked u of
+            Just (LockedGit "miso" repo _ c _) | not got, all (`notElem` planned) ds ->
+              fromPrebuilt db repo c (maybe "" id (depFlags d))
+            _ -> pure False
+          when pre $ do
+            info ("Installing " ++ what ++ " (built already, as it came with this miso)")
+            toStore db sid (depName d ++ "-" ++ maybe "" id (unitVersion u) ++ ".pkg") keyLines
+          pure (got || pre)
         _ -> pure False
       unless fromCache $ do
         info $ case uTool u of
@@ -335,7 +345,7 @@ sourceOf vend d ml =
       unless unpacked $ run ("tar -xzf " ++ quote tarball ++ " -C " ++ quote (home ++ "/src/hackage"))
       pure (Just dir, Nothing)
     (_, Just (LockedGit n repo _ c _)) -> do
-      dir <- gitCheckout n repo c
+      dir <- checkoutAt n repo c
       pure (Just dir, Nothing)
     (_, Nothing) -> case (depGit d, depPath d, depVersion d) of
       (_, Just dir, _) -> do
@@ -547,7 +557,8 @@ listPackages = do
 -- versions, and dropping those no longer needed.  miso update moves all of
 -- them to the versions of a new package list.
 lockDeps :: Config -> IO ()
-lockDeps cfg = do
+lockDeps cfg0 = do
+  cfg <- withMiso cfg0
   setupEnv
   old <- maybe [] id <$> readLock
   info ("Locking the dependencies in " ++ lockFile)
@@ -563,7 +574,8 @@ lockDeps cfg = do
 -- | The libraries of the tool the project uses (those it does not
 -- replace), as name-version, for miso.lock.
 usedToolLibraries :: Config -> IO [String]
-usedToolLibraries cfg = do
+usedToolLibraries cfg0 = do
+  cfg <- withMiso cfg0
   tool <- bundledPackages
   replaced <- filterM (isReplacement tool) (allDeps cfg)
   pure [ bName b ++ "-" ++ bVersion b | b <- tool, bSource b /= Nothing, bName b `notElem` map depName replaced ]
@@ -572,7 +584,8 @@ usedToolLibraries cfg = do
 -- snapshot), and every package locked again at its version, git
 -- dependencies at the commit of their rev now; then install them.
 updateDeps :: Config -> IO ()
-updateDeps cfg = do
+updateDeps cfg0 = do
+  cfg <- withMiso cfg0
   setupEnv
   info "Updating the package list"
   withGlobalCabal (runSelf "mcabal update")
@@ -647,7 +660,8 @@ vendoredDir vs ml d = do
 -- the packages they need.  Local dependencies, and the packages that come
 -- with the tool or with MicroHs, are not (they are not locked).
 vendorDeps :: Config -> IO ()
-vendorDeps cfg = do
+vendorDeps cfg0 = do
+  cfg <- withMiso cfg0
   lockDeps cfg
   lock <- maybe [] id <$> readLock
   old <- maybe [] id <$> readVendor
@@ -694,8 +708,12 @@ vendorDeps cfg = do
           run ("mv " ++ quote (tmp ++ "/" ++ n ++ "-" ++ v) ++ " " ++ quote dir)
         LockedGit n repo _ c _ -> do
           -- the files of the commit, without .git and what builds left there
-          src <- gitCheckout n repo c
-          run ("git -C " ++ quote src ++ " archive --format=tar " ++ quote c ++ " | tar -xf - -C " ++ quote tmp)
+          -- (the release's miso is its source, not a checkout)
+          src <- checkoutAt n repo c
+          isRepo <- doesDirectoryExist (src ++ "/.git")
+          if isRepo
+            then run ("git -C " ++ quote src ++ " archive --format=tar " ++ quote c ++ " | tar -xf - -C " ++ quote tmp)
+            else run ("cd " ++ quote src ++ " && tar -cf - --exclude=dist-mcabal . | tar -xf - -C " ++ quote tmp)
           run ("mv " ++ quote tmp ++ " " ++ quote dir)
       run ("rm -rf " ++ quote tmp)
       step ("+ " ++ dir ++ " " ++ describe l)

src/Miso/CLI/Doctor.hs

@@ -37,7 +37,11 @@ doctor = do
   pkgs <- if dbOk then listDirectory (db ++ "/packages") else pure []
   let hasPkg p = any ((p ++ "-") `isPrefixOf`) pkgs
   check (hasPkg "base") "base" "the package database is damaged; reinstall: miso upgrade"
-  check (hasPkg "miso") "miso" "the package database is damaged; reinstall: miso upgrade"
+  -- miso: prebuilt next to the packages (miso-prebuilt), or among them (a
+  -- release from before projects depended on miso)
+  hasDefault <- doesFileExist (db ++ "/miso-default")
+  prebuilt <- if hasDefault then doesDirectoryExist (db ++ "/miso-prebuilt/packages") else pure False
+  check (prebuilt || hasPkg "miso") "miso" "the package database is damaged; reinstall: miso upgrade"
   cpphs <- doesFileExist (parent exe ++ "/cpphs" ++ exeSuffix)
   check cpphs "cpphs (for modules with CPP)" ("missing next to " ++ exe ++ "; reinstall: miso upgrade")
   mhome <- misoHome

src/Miso/CLI/Lock.hs

@@ -6,7 +6,7 @@
 -- > # Local (path:) dependencies are not locked.
 -- > miso: 0.1.0
 -- > mhs: 0.16.10.0
--- > bundled: containers-0.8 miso-1.14.0.0 mtl-2.3.1 transformers-0.6.1.2
+-- > bundled: containers-0.8 mtl-2.3.1 transformers-0.6.1.2
 -- > packages:
 -- >   - name: split               # from Hackage: the version, and the
 -- >     hackage: 0.2.5.1          # sha256 of its tarball

src/Miso/CLI/Resolve.hs

@@ -18,6 +18,7 @@ module Miso.CLI.Resolve
   , packageDeps
   , mcabalEnv
   , isReplacement
+  , withMiso
   ) where
 
 import Control.Exception (SomeException, evaluate, try)
@@ -106,6 +107,36 @@ needsInstall env bundled n
     isOurs p = (n ++ "-") `isPrefixOf` p && ".pkg" `isSuffixOf'` p
                && all (`elem` "0123456789.") (drop (length n + 1) (take (length p - 4) p))
 
+-- | The project's dependencies, with miso as a package like the others: a
+-- dependency on miso with no source of its own (- miso) is the release's
+-- (Miso.CLI.Store), and a project whose code imports Miso but does not
+-- depend on it (miso.yml from a release that had miso among its packages)
+-- depends on it too, with a warning.  With such a release, as it is.
+withMiso :: Config -> IO Config
+withMiso cfg = do
+  md <- misoDefault
+  case md of
+    Nothing -> pure cfg
+    Just m -> do
+      let has = any ((== "miso") . depName) (allDeps cfg)
+      imports <- if has then pure False else importsMiso cfg
+      when imports $
+        warn ("the code imports Miso, but miso.yml has no dependency on miso; using the miso of this release "
+              ++ mdVersion m ++ " (miso add miso adds it)")
+      let plain d = depName d == "miso" && isHackage d && depVersion d == Nothing
+          fix ds = [ if plain d then defaultMisoDep m (depFlags d) else d | d <- ds ]
+      pure cfg { cfgDeps = fix (cfgDeps cfg) ++ [ defaultMisoDep m Nothing | imports ]
+               , cfgLibrary = fmap (\l -> l { libDeps = fix (libDeps l) }) (cfgLibrary cfg)
+               , cfgExecutables = [ e { exeDeps = fix (exeDeps e) } | e <- cfgExecutables cfg ]
+               , cfgTests = [ t { exeDeps = fix (exeDeps t) } | t <- cfgTests cfg ] }
+
+-- | Does the project's code import a module of miso?
+importsMiso :: Config -> IO Bool
+importsMiso cfg = do
+  dirs <- filterM doesDirectoryExist (allSourceDirs cfg)
+  if null dirs then pure False
+    else tryRun ("grep -rqE --include='*.hs' '^import +(qualified +)?Miso' " ++ unwords (map quote dirs))
+
 -- | The packages a package needs: the build-depends of its libraries and
 -- executables that are buildable, for this OS, architecture and compiler
 -- and the cabal flags given, with MicroHs's names, as mcabal reads them;
@@ -152,9 +183,15 @@ resolveTree keep old cfg = do
   -- a package of the tool that miso.yml replaces is locked as any other
   let bundled = [ bName b | b <- tool, bName b `notElem` map depName replaced ]
   home <- misoHome
+  md <- misoDefault
   state <- newIORef ([], [])            -- the packages seen, and the lock (reversed)
   let pn = patchedName env
       roots = allDeps cfg
+      -- the dependencies of miso.yml by that name; miso, when a package
+      -- needs it, is the release's
+      rootsFor n = case [ d | d <- roots, pn (depName d) == n ] of
+        [] | n == "miso", Just m <- md -> [defaultMisoDep m Nothing]
+        ds -> ds
       flagsOf d = maybe [] MC.decodeCabalFlags (depFlags d)
       emit l = modifyIORef state (\(seen, out) -> (seen, l : out))
       -- why a package is needed: "(needed by fail, needed by transformers-compat, ...)"
@@ -165,14 +202,18 @@ resolveTree keep old cfg = do
         (seen, _) <- readIORef state
         unless (n `elem` seen) $ do
           modifyIORef state (\(s, out) -> (n : s, out))
-          case [ d | d <- roots, pn (depName d) == n ] of
+          case rootsFor n of
             d : _ | Just dir <- depPath d -> do
               ok <- doesDirectoryExist dir
               unless ok $ failWith ("dependency " ++ depName d ++ ": no directory " ++ dir)
               packageDeps env (flagsOf d) (why chain) dir >>= mapM_ (visit (n : chain))
             d : _ | isGit d -> do
-              l <- maybe (resolveGit d) pure (if keep then lockedFor pn old d else Nothing)
-              dir <- gitCheckout (lkName l) (lkRepo l) (lkCommit l)
+              let resolve = case md of
+                    -- the release's miso, at its commit
+                    Just m | isDefaultMiso m d -> pure (LockedGit "miso" (mdRepo m) Nothing (mdCommit m) (Just (mdVersion m)))
+                    _ -> resolveGit d
+              l <- maybe resolve pure (if keep then lockedFor pn old d else Nothing)
+              dir <- checkoutAt (lkName l) (lkRepo l) (lkCommit l)
               packageDeps env (flagsOf d) (why chain) dir >>= mapM_ (visit (n : chain))
               emit l
             mroot -> do