normalise licence headers
[yaffs2.git] / yaffs_guts.c
index bc4e9b9846cf31dfb7b93e6387e1c13b9d658b14..f88851eea5c89e971120c6267b6a7e012f1bebe9 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * YAFFS: Yet another FFS. A NAND-flash specific file system. 
+ * YAFFS: Yet another FFS. A NAND-flash specific file system.
  *
  * Copyright (C) 2002 Aleph One Ltd.
  *   for Toby Churchill Ltd and Brightstar Engineering
@@ -13,7 +13,7 @@
  */
 
 const char *yaffs_guts_c_version =
-    "$Id: yaffs_guts.c,v 1.32 2006-05-08 10:13:34 charles Exp $";
+    "$Id: yaffs_guts.c,v 1.45 2006-11-14 03:07:17 charles Exp $";
 
 #include "yportenv.h"
 
@@ -22,11 +22,15 @@ const char *yaffs_guts_c_version =
 #include "yaffs_tagsvalidity.h"
 
 #include "yaffs_tagscompat.h"
+#ifndef CONFIG_YAFFS_OWN_SORT
+#include "yaffs_qsort.h"
+#endif
 #include "yaffs_nand.h"
 
 #include "yaffs_checkptrw.h"
 
 #include "yaffs_nand.h"
+#include "yaffs_packedtags2.h"
 
 
 #ifdef CONFIG_YAFFS_WINCE
@@ -41,7 +45,7 @@ void yfsd_UnlockYAFFS(BOOL fsLockOnly);
 
 /* Robustification (if it ever comes about...) */
 static void yaffs_RetireBlock(yaffs_Device * dev, int blockInNAND);
-static void yaffs_HandleWriteChunkError(yaffs_Device * dev, int chunkInNAND);
+static void yaffs_HandleWriteChunkError(yaffs_Device * dev, int chunkInNAND, int erasedOk);
 static void yaffs_HandleWriteChunkOk(yaffs_Device * dev, int chunkInNAND,
                                     const __u8 * data,
                                     const yaffs_ExtendedTags * tags);
@@ -50,6 +54,7 @@ static void yaffs_HandleUpdateChunk(yaffs_Device * dev, int chunkInNAND,
 
 /* Other local prototypes */
 static int yaffs_UnlinkObject( yaffs_Object *obj);
+static int yaffs_ObjectHasCachedWriteData(yaffs_Object *obj);
 
 static void yaffs_HardlinkFixup(yaffs_Device *dev, yaffs_Object *hardList);
 
@@ -89,7 +94,7 @@ static int yaffs_TagsMatch(const yaffs_ExtendedTags * tags, int objectId,
 
 loff_t yaffs_GetFileSize(yaffs_Object * obj);
 
-static int yaffs_AllocateChunk(yaffs_Device * dev, int useReserve);
+static int yaffs_AllocateChunk(yaffs_Device * dev, int useReserve, yaffs_BlockInfo **blockUsedPtr);
 
 static void yaffs_VerifyFreeChunks(yaffs_Device * dev);
 
@@ -104,6 +109,75 @@ static void yaffs_InvalidateChunkCache(yaffs_Object * object, int chunkId);
 
 static void yaffs_InvalidateCheckpoint(yaffs_Device *dev);
 
+
+
+/* Function to calculate chunk and offset */
+
+static void yaffs_AddrToChunk(yaffs_Device *dev, loff_t addr, __u32 *chunk, __u32 *offset)
+{
+       if(dev->chunkShift){
+               /* Easy-peasy power of 2 case */
+               *chunk  = (__u32)(addr >> dev->chunkShift);
+               *offset = (__u32)(addr & dev->chunkMask);
+       }
+       else if(dev->crumbsPerChunk)
+       {
+               /* Case where we're using "crumbs" */
+               *offset = (__u32)(addr & dev->crumbMask);
+               addr >>= dev->crumbShift;
+               *chunk = ((__u32)addr)/dev->crumbsPerChunk;
+               *offset += ((addr - (*chunk * dev->crumbsPerChunk)) << dev->crumbShift);
+       }
+       else
+               YBUG();
+}
+
+/* Function to return the number of shifts for a power of 2 greater than or equal 
+ * to the given number
+ * Note we don't try to cater for all possible numbers and this does not have to
+ * be hellishly efficient.
+ */
+static __u32 ShiftsGE(__u32 x)
+{
+       int extraBits;
+       int nShifts;
+       
+       nShifts = extraBits = 0;
+       
+       while(x>1){
+               if(x & 1) extraBits++;
+               x>>=1;
+               nShifts++;
+       }
+
+       if(extraBits) 
+               nShifts++;
+               
+       return nShifts;
+}
+
+/* Function to return the number of shifts to get a 1 in bit 0
+ */
+static __u32 ShiftDiv(__u32 x)
+{
+       int nShifts;
+       
+       nShifts =  0;
+       
+       if(!x) return 0;
+       
+       while( !(x&1)){
+               x>>=1;
+               nShifts++;
+       }
+               
+       return nShifts;
+}
+
+
+
 /* 
  * Temporary buffer manipulations.
  */
@@ -139,7 +213,7 @@ static __u8 *yaffs_GetTempBuffer(yaffs_Device * dev, int lineNo)
         */
 
        dev->unmanagedTempAllocations++;
-       return YMALLOC(dev->nBytesPerChunk);
+       return YMALLOC(dev->nDataBytesPerChunk);
 
 }
 
@@ -165,6 +239,32 @@ static void yaffs_ReleaseTempBuffer(yaffs_Device * dev, __u8 * buffer,
 
 }
 
+/*
+ * Determine if we have a managed buffer.
+ */
+int yaffs_IsManagedTempBuffer(yaffs_Device * dev, const __u8 * buffer)
+{
+       int i;
+       for (i = 0; i < YAFFS_N_TEMP_BUFFERS; i++) {
+               if (dev->tempBuffer[i].buffer == buffer)
+                       return 1;
+
+       }
+
+    for (i = 0; i < dev->nShortOpCaches; i++) {
+        if( dev->srCache[i].data == buffer )
+            return 1;
+
+    }
+
+    if (buffer == dev->checkpointBuffer)
+      return 1;
+
+    T(YAFFS_TRACE_ALWAYS,
+         (TSTR("yaffs: unmaged buffer detected.\n" TENDSTR)));
+    return 0;
+}
+
 /*
  * Chunk bitmap manipulations
  */
@@ -267,10 +367,15 @@ static int yaffs_CheckChunkErased(struct yaffs_DeviceStruct *dev,
        int retval = YAFFS_OK;
        __u8 *data = yaffs_GetTempBuffer(dev, __LINE__);
        yaffs_ExtendedTags tags;
+       int result;
 
-       yaffs_ReadChunkWithTagsFromNAND(dev, chunkInNAND, data, &tags);
+       result = yaffs_ReadChunkWithTagsFromNAND(dev, chunkInNAND, data, &tags);
+       
+       if(tags.eccResult > YAFFS_ECC_RESULT_NO_ERROR)
+               retval = YAFFS_FAIL;
+               
 
-       if (!yaffs_CheckFF(data, dev->nBytesPerChunk) || tags.chunkUsed) {
+       if (!yaffs_CheckFF(data, dev->nDataBytesPerChunk) || tags.chunkUsed) {
                T(YAFFS_TRACE_NANDACCESS,
                  (TSTR("Chunk %d not erased" TENDSTR), chunkInNAND));
                retval = YAFFS_FAIL;
@@ -289,42 +394,75 @@ static int yaffs_WriteNewChunkWithTagsToNAND(struct yaffs_DeviceStruct *dev,
 {
        int chunk;
 
-       int writeOk = 1;
+       int writeOk = 0;
+       int erasedOk = 1;
        int attempts = 0;
+       yaffs_BlockInfo *bi;
        
        yaffs_InvalidateCheckpoint(dev);
 
        do {
-               chunk = yaffs_AllocateChunk(dev, useReserve);
+               chunk = yaffs_AllocateChunk(dev, useReserve,&bi);
 
                if (chunk >= 0) {
+                       /* First check this chunk is erased, if it needs checking.
+                        * The checking policy (unless forced always on) is as follows:
+                        * Check the first page we try to write in a block.
+                        * - If the check passes then we don't need to check any more.
+                        * - If the check fails, we check again...
+                        * If the block has been erased, we don't need to check.
+                        *
+                        * However, if the block has been prioritised for gc, then
+                        * we think there might be something odd about this block
+                        * and stop using it.
+                        *
+                        * Rationale:
+                        * We should only ever see chunks that have not been erased
+                        * if there was a partially written chunk due to power loss
+                        * This checking policy should catch that case with very
+                        * few checks and thus save a lot of checks that are most likely not
+                        * needed.
+                        */
+                        
+                        if(bi->gcPrioritise){
+                                       yaffs_DeleteChunk(dev, chunk, 1, __LINE__);
+                       } else {
+#ifdef CONFIG_YAFFS_ALWAYS_CHECK_CHUNK_ERASED
+
+                               bi->skipErasedCheck = 0;
 
-                       /* First check this chunk is erased... */
-#ifndef CONFIG_YAFFS_DISABLE_CHUNK_ERASED_CHECK
-                       writeOk = yaffs_CheckChunkErased(dev, chunk);
 #endif
-                       if (!writeOk) {
-                               T(YAFFS_TRACE_ERROR,
-                                 (TSTR
-                                  ("**>> yaffs chunk %d was not erased"
-                                   TENDSTR), chunk));
-                       } else {
-                               writeOk =
-                                   yaffs_WriteChunkWithTagsToNAND(dev, chunk,
-                                                                  data, tags);
-                       }
-                       attempts++;
+                               if(!bi->skipErasedCheck){
+                                       erasedOk = yaffs_CheckChunkErased(dev, chunk);
+                                       if(erasedOk && !bi->gcPrioritise)
+                                               bi->skipErasedCheck = 1;
+                               }
 
-                       if (writeOk) {
-                               /*
-                                *  Copy the data into the robustification buffer.
-                                *  NB We do this at the end to prevent duplicates in the case of a write error.
-                                *  Todo
-                                */
-                               yaffs_HandleWriteChunkOk(dev, chunk, data,
-                                                        tags);
-                       } else {
-                               yaffs_HandleWriteChunkError(dev, chunk);
+                               if (!erasedOk) {
+                                       T(YAFFS_TRACE_ERROR,
+                                         (TSTR
+                                          ("**>> yaffs chunk %d was not erased"
+                                           TENDSTR), chunk));
+                               } else {
+                                       writeOk =
+                                           yaffs_WriteChunkWithTagsToNAND(dev, chunk,
+                                                                          data, tags);
+                               }
+                       
+                               attempts++;
+
+                               if (writeOk) {
+                                       /*
+                                        *  Copy the data into the robustification buffer.
+                                        *  NB We do this at the end to prevent duplicates in the case of a write error.
+                                        *  Todo
+                                        */
+                                       yaffs_HandleWriteChunkOk(dev, chunk, data, tags);
+                               
+                               } else {
+                                       /* The erased check or write failed */
+                                       yaffs_HandleWriteChunkError(dev, chunk, erasedOk);
+                               }
                        }
                }
 
@@ -346,13 +484,15 @@ static int yaffs_WriteNewChunkWithTagsToNAND(struct yaffs_DeviceStruct *dev,
  
 static void yaffs_RetireBlock(yaffs_Device * dev, int blockInNAND)
 {
+       yaffs_BlockInfo *bi = yaffs_GetBlockInfo(dev, blockInNAND);
 
        yaffs_InvalidateCheckpoint(dev);
        
        yaffs_MarkBlockBad(dev, blockInNAND);
 
-       yaffs_GetBlockInfo(dev, blockInNAND)->blockState =
-           YAFFS_BLOCK_STATE_DEAD;
+       bi->blockState = YAFFS_BLOCK_STATE_DEAD;
+       bi->gcPrioritise = 0;
+       bi->needsRetiring = 0;
 
        dev->nRetiredBlocks++;
 }
@@ -373,12 +513,55 @@ static void yaffs_HandleUpdateChunk(yaffs_Device * dev, int chunkInNAND,
 {
 }
 
-static void yaffs_HandleWriteChunkError(yaffs_Device * dev, int chunkInNAND)
+void yaffs_HandleChunkError(yaffs_Device *dev, yaffs_BlockInfo *bi)
+{
+       if(!bi->gcPrioritise){
+               bi->gcPrioritise = 1;
+               dev->hasPendingPrioritisedGCs = 1;
+               bi->chunkErrorStrikes ++;
+               
+               if(bi->chunkErrorStrikes > 3){
+                       bi->needsRetiring = 1; /* Too many stikes, so retire this */
+                       T(YAFFS_TRACE_ALWAYS, (TSTR("yaffs: Block struck out" TENDSTR)));
+
+               }
+               
+       }
+}
+
+static void yaffs_ReportOddballBlocks(yaffs_Device *dev)
 {
+       int i;
+               
+       for(i = dev->internalStartBlock; i <= dev->internalEndBlock && (yaffs_traceMask & YAFFS_TRACE_BAD_BLOCKS); i++){
+               yaffs_BlockInfo *bi = yaffs_GetBlockInfo(dev,i);
+               if(bi->needsRetiring || bi->gcPrioritise)
+                       T(YAFFS_TRACE_BAD_BLOCKS,(TSTR("yaffs block %d%s%s" TENDSTR),
+                               i,
+                               bi->needsRetiring ? " needs retiring" : "",
+                               bi->gcPrioritise ?  " gc prioritised" : ""));
+               
+       }
+}
+
+static void yaffs_HandleWriteChunkError(yaffs_Device * dev, int chunkInNAND, int erasedOk)
+{
+
        int blockInNAND = chunkInNAND / dev->nChunksPerBlock;
+       yaffs_BlockInfo *bi = yaffs_GetBlockInfo(dev, blockInNAND);
+
+       yaffs_HandleChunkError(dev,bi);
+               
+       
+       if(erasedOk ) {
+               /* Was an actual write failure, so mark the block for retirement  */
+               bi->needsRetiring = 1;
+               T(YAFFS_TRACE_ERROR | YAFFS_TRACE_BAD_BLOCKS,
+                 (TSTR("**>> Block %d needs retiring" TENDSTR), blockInNAND));
 
-       /* Mark the block for retirement */
-       yaffs_GetBlockInfo(dev, blockInNAND)->needsRetiring = 1;
+               
+       }
+       
        /* Delete the chunk */
        yaffs_DeleteChunk(dev, chunkInNAND, 1, __LINE__);
 }
@@ -1722,6 +1905,15 @@ int yaffs_RenameObject(yaffs_Object * oldDir, const YCHAR * oldName,
 #endif
 
        obj = yaffs_FindObjectByName(oldDir, oldName);
+       /* Check new name to long. */
+       if (obj->variantType == YAFFS_OBJECT_TYPE_SYMLINK &&
+           yaffs_strlen(newName) > YAFFS_MAX_ALIAS_LENGTH)
+         /* ENAMETOOLONG */
+         return YAFFS_FAIL;
+       else if (obj->variantType != YAFFS_OBJECT_TYPE_SYMLINK &&
+                yaffs_strlen(newName) > YAFFS_MAX_NAME_LENGTH)
+         /* ENAMETOOLONG */
+         return YAFFS_FAIL;
 
        if (obj && obj->renameAllowed) {
 
@@ -1765,7 +1957,7 @@ static int yaffs_InitialiseBlocks(yaffs_Device * dev)
                dev->blockInfoAlt = 0;
        
        /* Set up dynamic blockinfo stuff. */
-       dev->chunkBitmapStride = (dev->nChunksPerBlock + 7) / 8; // round up bytes
+       dev->chunkBitmapStride = (dev->nChunksPerBlock + 7) / 8; /* round up bytes */
        dev->chunkBits = YMALLOC(dev->chunkBitmapStride * nBlocks);
        if(!dev->chunkBits){
                dev->chunkBits = YMALLOC_ALT(dev->chunkBitmapStride * nBlocks);
@@ -1838,8 +2030,6 @@ static int yaffs_BlockNotDisqualifiedFromGC(yaffs_Device * dev,
         */
        return (bi->sequenceNumber <= dev->oldestDirtySequence);
 
-       return 1;
-
 }
 
 /* FindDiretiestBlock is used to select the dirtiest block (or close enough)
@@ -1856,8 +2046,31 @@ static int yaffs_FindBlockForGarbageCollection(yaffs_Device * dev,
        int iterations;
        int dirtiest = -1;
        int pagesInUse;
+       int prioritised=0;
        yaffs_BlockInfo *bi;
        static int nonAggressiveSkip = 0;
+       int pendingPrioritisedExist = 0;
+       
+       /* First let's see if we need to grab a prioritised block */
+       if(dev->hasPendingPrioritisedGCs){
+               for(i = dev->internalStartBlock; i < dev->internalEndBlock && !prioritised; i++){
+
+                       bi = yaffs_GetBlockInfo(dev, i);
+                       if(bi->gcPrioritise) {
+                               pendingPrioritisedExist = 1;
+                               if(bi->blockState == YAFFS_BLOCK_STATE_FULL &&
+                                  yaffs_BlockNotDisqualifiedFromGC(dev, bi)){
+                                       pagesInUse = (bi->pagesInUse - bi->softDeletions);
+                                       dirtiest = i;
+                                       prioritised = 1;
+                                       aggressive = 1; /* Fool the non-aggressive skip logiv below */
+                               }
+                       }
+               }
+               
+               if(!pendingPrioritisedExist) /* None found, so we can clear this */
+                       dev->hasPendingPrioritisedGCs = 0;
+       }
 
        /* If we're doing aggressive GC then we are happy to take a less-dirty block, and
         * search harder.
@@ -1871,8 +2084,9 @@ static int yaffs_FindBlockForGarbageCollection(yaffs_Device * dev,
                return -1;
        }
 
-       pagesInUse =
-           (aggressive) ? dev->nChunksPerBlock : YAFFS_PASSIVE_GC_CHUNKS + 1;
+       if(!prioritised)
+               pagesInUse =
+                       (aggressive) ? dev->nChunksPerBlock : YAFFS_PASSIVE_GC_CHUNKS + 1;
 
        if (aggressive) {
                iterations =
@@ -1886,7 +2100,7 @@ static int yaffs_FindBlockForGarbageCollection(yaffs_Device * dev,
                }
        }
 
-       for (i = 0; i <= iterations && pagesInUse > 0; i++) {
+       for (i = 0; i <= iterations && pagesInUse > 0 && !prioritised; i++) {
                b++;
                if (b < dev->internalStartBlock || b > dev->internalEndBlock) {
                        b = dev->internalStartBlock;
@@ -1900,9 +2114,17 @@ static int yaffs_FindBlockForGarbageCollection(yaffs_Device * dev,
 
                bi = yaffs_GetBlockInfo(dev, b);
 
+#if 0
+               if (bi->blockState == YAFFS_BLOCK_STATE_CHECKPOINT) {
+                       dirtiest = b;
+                       pagesInUse = 0;
+               }
+               else 
+#endif
+
                if (bi->blockState == YAFFS_BLOCK_STATE_FULL &&
-                   (bi->pagesInUse - bi->softDeletions) < pagesInUse &&
-                   yaffs_BlockNotDisqualifiedFromGC(dev, bi)) {
+                      (bi->pagesInUse - bi->softDeletions) < pagesInUse &&
+                       yaffs_BlockNotDisqualifiedFromGC(dev, bi)) {
                        dirtiest = b;
                        pagesInUse = (bi->pagesInUse - bi->softDeletions);
                }
@@ -1912,8 +2134,8 @@ static int yaffs_FindBlockForGarbageCollection(yaffs_Device * dev,
 
        if (dirtiest > 0) {
                T(YAFFS_TRACE_GC,
-                 (TSTR("GC Selected block %d with %d free" TENDSTR), dirtiest,
-                  dev->nChunksPerBlock - pagesInUse));
+                 (TSTR("GC Selected block %d with %d free, prioritised:%d" TENDSTR), dirtiest,
+                  dev->nChunksPerBlock - pagesInUse,prioritised));
        }
 
        dev->oldestDirtySequence = 0;
@@ -1934,6 +2156,11 @@ static void yaffs_BlockBecameDirty(yaffs_Device * dev, int blockNo)
        /* If the block is still healthy erase it and mark as clean.
         * If the block has had a data failure, then retire it.
         */
+        
+       T(YAFFS_TRACE_GC | YAFFS_TRACE_ERASE,
+               (TSTR("yaffs_BlockBecameDirty block %d state %d %s"TENDSTR),
+               blockNo, bi->blockState, (bi->needsRetiring) ? "needs retiring" : ""));
+               
        bi->blockState = YAFFS_BLOCK_STATE_DIRTY;
 
        if (!bi->needsRetiring) {
@@ -1966,6 +2193,8 @@ static void yaffs_BlockBecameDirty(yaffs_Device * dev, int blockNo)
                bi->pagesInUse = 0;
                bi->softDeletions = 0;
                bi->hasShrinkHeader = 0;
+               bi->skipErasedCheck = 1;  /* This is clean, so no need to check */
+               bi->gcPrioritise = 0;
                yaffs_ClearChunkBits(dev, blockNo);
 
                T(YAFFS_TRACE_ERASE,
@@ -2028,14 +2257,24 @@ static int yaffs_FindBlockForAllocation(yaffs_Device * dev)
 }
 
 
-
+// Check if there's space to allocate...
+// Thinks.... do we need top make this ths same as yaffs_GetFreeChunks()?
 static int yaffs_CheckSpaceForAllocation(yaffs_Device * dev)
 {
-       int reservedChunks = (dev->nReservedBlocks * dev->nChunksPerBlock);
+       int reservedChunks;
+       int reservedBlocks = dev->nReservedBlocks;
+       int checkpointBlocks;
+       
+       checkpointBlocks =  dev->nCheckpointReservedBlocks - dev->blocksInCheckpoint;
+       if(checkpointBlocks < 0)
+               checkpointBlocks = 0;
+       
+       reservedChunks = ((reservedBlocks + checkpointBlocks) * dev->nChunksPerBlock);
+       
        return (dev->nFreeChunks > reservedChunks);
 }
 
-static int yaffs_AllocateChunk(yaffs_Device * dev, int useReserve)
+static int yaffs_AllocateChunk(yaffs_Device * dev, int useReserve, yaffs_BlockInfo **blockUsedPtr)
 {
        int retVal;
        yaffs_BlockInfo *bi;
@@ -2076,6 +2315,9 @@ static int yaffs_AllocateChunk(yaffs_Device * dev, int useReserve)
                        dev->allocationBlock = -1;
                }
 
+               if(blockUsedPtr)
+                       *blockUsedPtr = bi;
+                       
                return retVal;
        }
        
@@ -2108,6 +2350,7 @@ static int yaffs_GarbageCollectBlock(yaffs_Device * dev, int block)
        int retVal = YAFFS_OK;
        int cleanups = 0;
        int i;
+       int isCheckpointBlock;
 
        int chunksBefore = yaffs_GetErasedChunks(dev);
        int chunksAfter;
@@ -2118,6 +2361,8 @@ static int yaffs_GarbageCollectBlock(yaffs_Device * dev, int block)
 
        yaffs_Object *object;
 
+       isCheckpointBlock = (bi->blockState == YAFFS_BLOCK_STATE_CHECKPOINT);
+       
        bi->blockState = YAFFS_BLOCK_STATE_COLLECTING;
 
        T(YAFFS_TRACE_TRACING,
@@ -2135,7 +2380,8 @@ static int yaffs_GarbageCollectBlock(yaffs_Device * dev, int block)
 
        dev->isDoingGC = 1;
 
-       if (!yaffs_StillSomeChunkBits(dev, block)) {
+       if (isCheckpointBlock ||
+           !yaffs_StillSomeChunkBits(dev, block)) {
                T(YAFFS_TRACE_TRACING,
                  (TSTR
                   ("Collecting block %d that has no chunks in use" TENDSTR),
@@ -2314,6 +2560,8 @@ static int yaffs_CheckGarbageCollection(yaffs_Device * dev)
        int aggressive;
        int gcOk = YAFFS_OK;
        int maxTries = 0;
+       
+       int checkpointBlockAdjust;
 
        if (dev->isDoingGC) {
                /* Bail out so we don't get recursive gc */
@@ -2326,7 +2574,12 @@ static int yaffs_CheckGarbageCollection(yaffs_Device * dev)
 
        do {
                maxTries++;
-               if (dev->nErasedBlocks < dev->nReservedBlocks) {
+               
+               checkpointBlockAdjust = (dev->nCheckpointReservedBlocks - dev->blocksInCheckpoint);
+               if(checkpointBlockAdjust < 0)
+                       checkpointBlockAdjust = 0;
+
+               if (dev->nErasedBlocks < (dev->nReservedBlocks + checkpointBlockAdjust)) {
                        /* We need a block soon...*/
                        aggressive = 1;
                } else {
@@ -2466,7 +2719,7 @@ static int yaffs_CheckFileSanity(yaffs_Object * in)
        objId = in->objectId;
        fSize = in->variant.fileVariant.fileSize;
        nChunks =
-           (fSize + in->myDev->nBytesPerChunk - 1) / in->myDev->nBytesPerChunk;
+           (fSize + in->myDev->nDataBytesPerChunk - 1) / in->myDev->nDataBytesPerChunk;
 
        for (chunk = 1; chunk <= nChunks; chunk++) {
                tn = yaffs_FindLevel0Tnode(in->myDev, &in->variant.fileVariant,
@@ -2633,13 +2886,13 @@ static int yaffs_ReadChunkDataFromObject(yaffs_Object * in, int chunkInInode,
 
        if (chunkInNAND >= 0) {
                return yaffs_ReadChunkWithTagsFromNAND(in->myDev, chunkInNAND,
-                                                      buffer, NULL);
+                                                      buffer,NULL);
        } else {
                T(YAFFS_TRACE_NANDACCESS,
                  (TSTR("Chunk %d not found zero instead" TENDSTR),
                   chunkInNAND));
                /* get sane (zero) data if you read a hole */
-               memset(buffer, 0, in->myDev->nBytesPerChunk);   
+               memset(buffer, 0, in->myDev->nDataBytesPerChunk);       
                return 0;
        }
 
@@ -2765,6 +3018,7 @@ int yaffs_UpdateObjectHeader(yaffs_Object * in, const YCHAR * name, int force,
 
        int prevChunkId;
        int retVal = 0;
+       int result = 0;
 
        int newChunkId;
        yaffs_ExtendedTags newTags;
@@ -2784,12 +3038,12 @@ int yaffs_UpdateObjectHeader(yaffs_Object * in, const YCHAR * name, int force,
                prevChunkId = in->chunkId;
 
                if (prevChunkId >= 0) {
-                       yaffs_ReadChunkWithTagsFromNAND(dev, prevChunkId,
+                       result = yaffs_ReadChunkWithTagsFromNAND(dev, prevChunkId,
                                                        buffer, NULL);
                        memcpy(oldName, oh->name, sizeof(oh->name));
                }
 
-               memset(buffer, 0xFF, dev->nBytesPerChunk);
+               memset(buffer, 0xFF, dev->nDataBytesPerChunk);
 
                oh->type = in->variantType;
                oh->yst_mode = in->yst_mode;
@@ -2887,7 +3141,8 @@ int yaffs_UpdateObjectHeader(yaffs_Object * in, const YCHAR * name, int force,
                                                  __LINE__);
                        }
 
-                       in->dirty = 0;
+                       if(!yaffs_ObjectHasCachedWriteData(in))
+                               in->dirty = 0;
 
                        /* If this was a shrink, then mark the block that the chunk lives on */
                        if (isShrink) {
@@ -2921,6 +3176,24 @@ int yaffs_UpdateObjectHeader(yaffs_Object * in, const YCHAR * name, int force,
  *   need a very intelligent search.
  */
 
+static int yaffs_ObjectHasCachedWriteData(yaffs_Object *obj)
+{
+       yaffs_Device *dev = obj->myDev;
+       int i;
+       yaffs_ChunkCache *cache;
+       int nCaches = obj->myDev->nShortOpCaches;
+       
+       for(i = 0; i < nCaches; i++){
+               cache = &dev->srCache[i];
+               if (cache->object == obj &&
+                   cache->dirty)
+                       return 1;
+       }
+       
+       return 0;
+}
+
+
 static void yaffs_FlushFilesChunkCache(yaffs_Object * obj)
 {
        yaffs_Device *dev = obj->myDev;
@@ -3178,7 +3451,7 @@ static int yaffs_WriteCheckpointValidityMarker(yaffs_Device *dev,int head)
        yaffs_CheckpointValidity cp;
        cp.structType = sizeof(cp);
        cp.magic = YAFFS_MAGIC;
-       cp.version = 1;
+       cp.version = YAFFS_CHECKPOINT_VERSION;
        cp.head = (head) ? 1 : 0;
        
        return (yaffs_CheckpointWrite(dev,&cp,sizeof(cp)) == sizeof(cp))?
@@ -3195,7 +3468,7 @@ static int yaffs_ReadCheckpointValidityMarker(yaffs_Device *dev, int head)
        if(ok)
                ok = (cp.structType == sizeof(cp)) &&
                     (cp.magic == YAFFS_MAGIC) &&
-                    (cp.version == 1) &&
+                    (cp.version == YAFFS_CHECKPOINT_VERSION) &&
                     (cp.head == ((head) ? 1 : 0));
        return ok ? 1 : 0;
 }
@@ -3360,13 +3633,15 @@ static int yaffs_CheckpointTnodeWorker(yaffs_Object * in, yaffs_Tnode * tn,
 {
        int i;
        yaffs_Device *dev = in->myDev;
+       int ok = 1;
+       int nTnodeBytes = (dev->tnodeWidth * YAFFS_NTNODES_LEVEL0)/8;
 
        if (tn) {
                if (level > 0) {
 
-                       for (i = 0; i < YAFFS_NTNODES_INTERNAL; i++){
+                       for (i = 0; i < YAFFS_NTNODES_INTERNAL && ok; i++){
                                if (tn->internal[i]) {
-                                       yaffs_CheckpointTnodeWorker(in,
+                                       ok = yaffs_CheckpointTnodeWorker(in,
                                                        tn->internal[i],
                                                        level - 1,
                                                        (chunkOffset<<YAFFS_TNODES_INTERNAL_BITS) + i);
@@ -3375,19 +3650,20 @@ static int yaffs_CheckpointTnodeWorker(yaffs_Object * in, yaffs_Tnode * tn,
                } else if (level == 0) {
                        __u32 baseOffset = chunkOffset <<  YAFFS_TNODES_LEVEL0_BITS;
                        /* printf("write tnode at %d\n",baseOffset); */
-                       yaffs_CheckpointWrite(dev,&baseOffset,sizeof(baseOffset));
-                       yaffs_CheckpointWrite(dev,tn,(dev->tnodeWidth * YAFFS_NTNODES_LEVEL0)/8);
+                       ok = (yaffs_CheckpointWrite(dev,&baseOffset,sizeof(baseOffset)) == sizeof(baseOffset));
+                       if(ok)
+                               ok = (yaffs_CheckpointWrite(dev,tn,nTnodeBytes) == nTnodeBytes);
                }
        }
 
-       return 1;
+       return ok;
 
 }
 
 static int yaffs_WriteCheckpointTnodes(yaffs_Object *obj)
 {
        __u32 endMarker = ~0;
-       int ok;
+       int ok = 1;
        
        if(obj->variantType == YAFFS_OBJECT_TYPE_FILE){
                ok = yaffs_CheckpointTnodeWorker(obj,
@@ -3459,7 +3735,11 @@ static int yaffs_WriteCheckpointObjects(yaffs_Device *dev)
                                if (!obj->deferedFree) {
                                        yaffs_ObjectToCheckpointObject(&cp,obj);
                                        cp.structType = sizeof(cp);
-                                       /* printf("Write out object %d type %d\n",obj->objectId,obj->variantType); */
+
+                                       T(YAFFS_TRACE_CHECKPOINT,(
+                                               TSTR("Checkpoint write object %d parent %d type %d chunk %d obj addr %x" TENDSTR),
+                                               cp.objectId,cp.parentId,cp.variantType,cp.chunkId,(unsigned) obj));
+                                               
                                        ok = (yaffs_CheckpointWrite(dev,&cp,sizeof(cp)) == sizeof(cp));
                                        
                                        if(ok && obj->variantType == YAFFS_OBJECT_TYPE_FILE){
@@ -3497,13 +3777,14 @@ static int yaffs_ReadCheckpointObjects(yaffs_Device *dev)
                        
                if(ok && cp.objectId == ~0)
                        done = 1;
-               else {
-                       /* printf("Read object %d type %d\n",cp.objectId,cp.variantType); */
+               else if(ok){
                        obj = yaffs_FindOrCreateObjectByNumber(dev,cp.objectId, cp.variantType);
+                       T(YAFFS_TRACE_CHECKPOINT,(TSTR("Checkpoint read object %d parent %d type %d chunk %d obj addr %x" TENDSTR),
+                               cp.objectId,cp.parentId,cp.variantType,cp.chunkId,(unsigned) obj));
                        if(obj) {
                                yaffs_CheckpointObjectToObject(obj,&cp);
                                if(obj->variantType == YAFFS_OBJECT_TYPE_FILE) {
-                                       yaffs_ReadCheckpointTnodes(obj);
+                                       ok = yaffs_ReadCheckpointTnodes(obj);
                                } else if(obj->variantType == YAFFS_OBJECT_TYPE_HARDLINK) {
                                        obj->hardLinks.next =
                                                    (struct list_head *)
@@ -3537,8 +3818,9 @@ static int yaffs_WriteCheckpointData(yaffs_Device *dev)
        if(ok)
                ok = yaffs_WriteCheckpointValidityMarker(dev,0);
                
-       yaffs_CheckpointClose(dev);
-               
+       if(!yaffs_CheckpointClose(dev))
+                ok = 0;
+                
        if(ok)
                dev->isCheckpointed = 1;
         else 
@@ -3562,39 +3844,57 @@ static int yaffs_ReadCheckpointData(yaffs_Device *dev)
        if(ok)
                ok = yaffs_ReadCheckpointValidityMarker(dev,0);
                
-               
+
+
+       if(!yaffs_CheckpointClose(dev))
+               ok = 0;
+
        if(ok)
                dev->isCheckpointed = 1;
         else 
                dev->isCheckpointed = 0;
 
-       yaffs_CheckpointClose(dev);
-
        return ok ? 1 : 0;
 
 }
 
 static void yaffs_InvalidateCheckpoint(yaffs_Device *dev)
 {
-       if(dev->isCheckpointed){
+       if(dev->isCheckpointed || 
+          dev->blocksInCheckpoint > 0){
                dev->isCheckpointed = 0;
                yaffs_CheckpointInvalidateStream(dev);
+               if(dev->superBlock && dev->markSuperBlockDirty)
+                       dev->markSuperBlockDirty(dev->superBlock);
        }
 }
 
 
 int yaffs_CheckpointSave(yaffs_Device *dev)
 {
+       yaffs_ReportOddballBlocks(dev);
+       T(YAFFS_TRACE_CHECKPOINT,(TSTR("save entry: isCheckpointed %d"TENDSTR),dev->isCheckpointed));
+
        if(!dev->isCheckpointed)
                yaffs_WriteCheckpointData(dev);
        
+       T(YAFFS_TRACE_CHECKPOINT,(TSTR("save exit: isCheckpointed %d"TENDSTR),dev->isCheckpointed));
+
        return dev->isCheckpointed;
 }
 
 int yaffs_CheckpointRestore(yaffs_Device *dev)
 {
+       int retval;
+       T(YAFFS_TRACE_CHECKPOINT,(TSTR("restore entry: isCheckpointed %d"TENDSTR),dev->isCheckpointed));
+       
+       retval = yaffs_ReadCheckpointData(dev);
+
+       T(YAFFS_TRACE_CHECKPOINT,(TSTR("restore exit: isCheckpointed %d"TENDSTR),dev->isCheckpointed));
+       
+       yaffs_ReportOddballBlocks(dev);
        
-       return yaffs_ReadCheckpointData(dev);
+       return retval;
 }
 
 /*--------------------- File read/write ------------------------
@@ -3607,7 +3907,7 @@ int yaffs_CheckpointRestore(yaffs_Device *dev)
  * Curve-balls: the first chunk might also be the last chunk.
  */
 
-int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, __u32 offset,
+int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, loff_t offset,
                           int nBytes)
 {
 
@@ -3623,16 +3923,18 @@ int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, __u32 offset,
        dev = in->myDev;
 
        while (n > 0) {
-               chunk = offset / dev->nBytesPerChunk + 1;   /* The first chunk is 1 */
-               start = offset % dev->nBytesPerChunk;
+               //chunk = offset / dev->nDataBytesPerChunk + 1;
+               //start = offset % dev->nDataBytesPerChunk;
+               yaffs_AddrToChunk(dev,offset,&chunk,&start);
+               chunk++;
 
                /* OK now check for the curveball where the start and end are in
                 * the same chunk.      
                 */
-               if ((start + n) < dev->nBytesPerChunk) {
+               if ((start + n) < dev->nDataBytesPerChunk) {
                        nToCopy = n;
                } else {
-                       nToCopy = dev->nBytesPerChunk - start;
+                       nToCopy = dev->nDataBytesPerChunk - start;
                }
 
                cache = yaffs_FindChunkCache(in, chunk);
@@ -3641,7 +3943,7 @@ int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, __u32 offset,
                 * then use the cache (if there is caching)
                 * else bypass the cache.
                 */
-               if (cache || nToCopy != dev->nBytesPerChunk) {
+               if (cache || nToCopy != dev->nDataBytesPerChunk) {
                        if (dev->nShortOpCaches > 0) {
 
                                /* If we can't find the data in the cache, then load it up. */
@@ -3702,7 +4004,7 @@ int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, __u32 offset,
 #ifdef CONFIG_YAFFS_WINCE
                        yfsd_UnlockYAFFS(TRUE);
 #endif
-                       memcpy(buffer, localBuffer, dev->nBytesPerChunk);
+                       memcpy(buffer, localBuffer, dev->nDataBytesPerChunk);
 
 #ifdef CONFIG_YAFFS_WINCE
                        yfsd_LockYAFFS(TRUE);
@@ -3725,7 +4027,7 @@ int yaffs_ReadDataFromFile(yaffs_Object * in, __u8 * buffer, __u32 offset,
        return nDone;
 }
 
-int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
+int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, loff_t offset,
                          int nBytes, int writeThrough)
 {
 
@@ -3744,14 +4046,16 @@ int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
        dev = in->myDev;
 
        while (n > 0 && chunkWritten >= 0) {
-               chunk = offset / dev->nBytesPerChunk + 1;
-               start = offset % dev->nBytesPerChunk;
+               //chunk = offset / dev->nDataBytesPerChunk + 1;
+               //start = offset % dev->nDataBytesPerChunk;
+               yaffs_AddrToChunk(dev,offset,&chunk,&start);
+               chunk++;
 
                /* OK now check for the curveball where the start and end are in
                 * the same chunk.
                 */
 
-               if ((start + n) < dev->nBytesPerChunk) {
+               if ((start + n) < dev->nDataBytesPerChunk) {
                        nToCopy = n;
 
                        /* Now folks, to calculate how many bytes to write back....
@@ -3761,10 +4065,10 @@ int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
 
                        nBytesRead =
                            in->variant.fileVariant.fileSize -
-                           ((chunk - 1) * dev->nBytesPerChunk);
+                           ((chunk - 1) * dev->nDataBytesPerChunk);
 
-                       if (nBytesRead > dev->nBytesPerChunk) {
-                               nBytesRead = dev->nBytesPerChunk;
+                       if (nBytesRead > dev->nDataBytesPerChunk) {
+                               nBytesRead = dev->nDataBytesPerChunk;
                        }
 
                        nToWriteBack =
@@ -3772,16 +4076,17 @@ int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
                             (start + n)) ? nBytesRead : (start + n);
 
                } else {
-                       nToCopy = dev->nBytesPerChunk - start;
-                       nToWriteBack = dev->nBytesPerChunk;
+                       nToCopy = dev->nDataBytesPerChunk - start;
+                       nToWriteBack = dev->nDataBytesPerChunk;
                }
 
-               if (nToCopy != dev->nBytesPerChunk) {
+               if (nToCopy != dev->nDataBytesPerChunk) {
                        /* An incomplete start or end chunk (or maybe both start and end chunk) */
                        if (dev->nShortOpCaches > 0) {
                                yaffs_ChunkCache *cache;
                                /* If we can't find the data in the cache, then load the cache */
                                cache = yaffs_FindChunkCache(in, chunk);
+                               
                                if (!cache
                                    && yaffs_CheckSpaceForAllocation(in->
                                                                     myDev)) {
@@ -3794,6 +4099,14 @@ int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
                                                                      cache->
                                                                      data);
                                }
+                               else if(cache && 
+                                       !cache->dirty &&
+                                       !yaffs_CheckSpaceForAllocation(in->myDev)){
+                                       /* Drop the cache if it was a read cache item and
+                                        * no space check has been made for it.
+                                        */ 
+                                        cache = NULL;
+                               }
 
                                if (cache) {
                                        yaffs_UseChunkCache(dev, cache, 1);
@@ -3865,20 +4178,20 @@ int yaffs_WriteDataToFile(yaffs_Object * in, const __u8 * buffer, __u32 offset,
 #ifdef CONFIG_YAFFS_WINCE
                        yfsd_UnlockYAFFS(TRUE);
 #endif
-                       memcpy(localBuffer, buffer, dev->nBytesPerChunk);
+                       memcpy(localBuffer, buffer, dev->nDataBytesPerChunk);
 #ifdef CONFIG_YAFFS_WINCE
                        yfsd_LockYAFFS(TRUE);
 #endif
                        chunkWritten =
                            yaffs_WriteChunkDataToObject(in, chunk, localBuffer,
-                                                        dev->nBytesPerChunk,
+                                                        dev->nDataBytesPerChunk,
                                                         0);
                        yaffs_ReleaseTempBuffer(dev, localBuffer, __LINE__);
 #else
                        /* A full chunk. Write directly from the supplied buffer. */
                        chunkWritten =
                            yaffs_WriteChunkDataToObject(in, chunk, buffer,
-                                                        dev->nBytesPerChunk,
+                                                        dev->nDataBytesPerChunk,
                                                         0);
 #endif
                        /* Since we've overwritten the cached data, we better invalidate it. */
@@ -3914,10 +4227,10 @@ static void yaffs_PruneResizedChunks(yaffs_Object * in, int newSize)
        yaffs_Device *dev = in->myDev;
        int oldFileSize = in->variant.fileVariant.fileSize;
 
-       int lastDel = 1 + (oldFileSize - 1) / dev->nBytesPerChunk;
+       int lastDel = 1 + (oldFileSize - 1) / dev->nDataBytesPerChunk;
 
-       int startDel = 1 + (newSize + dev->nBytesPerChunk - 1) /
-           dev->nBytesPerChunk;
+       int startDel = 1 + (newSize + dev->nDataBytesPerChunk - 1) /
+           dev->nDataBytesPerChunk;
        int i;
        int chunkId;
 
@@ -3949,14 +4262,16 @@ static void yaffs_PruneResizedChunks(yaffs_Object * in, int newSize)
 
 }
 
-int yaffs_ResizeFile(yaffs_Object * in, int newSize)
+int yaffs_ResizeFile(yaffs_Object * in, loff_t newSize)
 {
 
        int oldFileSize = in->variant.fileVariant.fileSize;
-       int sizeOfPartialChunk;
+       int newSizeOfPartialChunk;
+       int newFullChunks;
+       
        yaffs_Device *dev = in->myDev;
-
-       sizeOfPartialChunk = newSize % dev->nBytesPerChunk;
+       
+       yaffs_AddrToChunk(dev, newSize, &newFullChunks, &newSizeOfPartialChunk);
 
        yaffs_FlushFilesChunkCache(in);
        yaffs_InvalidateWholeChunkCache(in);
@@ -3975,19 +4290,20 @@ int yaffs_ResizeFile(yaffs_Object * in, int newSize)
 
                yaffs_PruneResizedChunks(in, newSize);
 
-               if (sizeOfPartialChunk != 0) {
-                       int lastChunk = 1 + newSize / dev->nBytesPerChunk;
+               if (newSizeOfPartialChunk != 0) {
+                       int lastChunk = 1 + newFullChunks;
+                       
                        __u8 *localBuffer = yaffs_GetTempBuffer(dev, __LINE__);
 
                        /* Got to read and rewrite the last chunk with its new size and zero pad */
                        yaffs_ReadChunkDataFromObject(in, lastChunk,
                                                      localBuffer);
 
-                       memset(localBuffer + sizeOfPartialChunk, 0,
-                              dev->nBytesPerChunk - sizeOfPartialChunk);
+                       memset(localBuffer + newSizeOfPartialChunk, 0,
+                              dev->nDataBytesPerChunk - newSizeOfPartialChunk);
 
                        yaffs_WriteChunkDataToObject(in, lastChunk, localBuffer,
-                                                    sizeOfPartialChunk, 1);
+                                                    newSizeOfPartialChunk, 1);
 
                        yaffs_ReleaseTempBuffer(dev, localBuffer, __LINE__);
                }
@@ -4344,6 +4660,18 @@ static void yaffs_HardlinkFixup(yaffs_Device *dev, yaffs_Object *hardList)
 
 
 
+static int ybicmp(const void *a, const void *b){
+    register int aseq = ((yaffs_BlockIndex *)a)->seq;
+    register int bseq = ((yaffs_BlockIndex *)b)->seq;
+    register int ablock = ((yaffs_BlockIndex *)a)->block;
+    register int bblock = ((yaffs_BlockIndex *)b)->block;
+    if( aseq == bseq )
+        return ablock - bblock;
+    else
+        return aseq - bseq;
+
+}
+
 static int yaffs_Scan(yaffs_Device * dev)
 {
        yaffs_ExtendedTags tags;
@@ -4352,6 +4680,7 @@ static int yaffs_Scan(yaffs_Device * dev)
        int startIterator;
        int endIterator;
        int nBlocksToScan = 0;
+       int result;
 
        int chunk;
        int c;
@@ -4370,6 +4699,14 @@ static int yaffs_Scan(yaffs_Device * dev)
 
        yaffs_BlockIndex *blockIndex = NULL;
 
+       if (dev->isYaffs2) {
+               T(YAFFS_TRACE_SCAN,
+                 (TSTR("yaffs_Scan is not for YAFFS2!" TENDSTR)));
+               return YAFFS_FAIL;
+       }
+       
+       //TODO  Throw all the yaffs2 stuuf out of yaffs_Scan since it is only for yaffs1 format.
+       
        T(YAFFS_TRACE_SCAN,
          (TSTR("yaffs_Scan starts  intstartblk %d intendblk %d..." TENDSTR),
           dev->internalStartBlock, dev->internalEndBlock));
@@ -4482,7 +4819,7 @@ static int yaffs_Scan(yaffs_Device * dev)
                        /* Read the tags and decide what to do */
                        chunk = blk * dev->nChunksPerBlock + c;
 
-                       yaffs_ReadChunkWithTagsFromNAND(dev, chunk, NULL,
+                       result = yaffs_ReadChunkWithTagsFromNAND(dev, chunk, NULL,
                                                        &tags);
 
                        /* Let's have a good look at this chunk... */
@@ -4548,7 +4885,7 @@ static int yaffs_Scan(yaffs_Device * dev)
                                yaffs_PutChunkIntoFile(in, tags.chunkId, chunk,
                                                       1);
                                endpos =
-                                   (tags.chunkId - 1) * dev->nBytesPerChunk +
+                                   (tags.chunkId - 1) * dev->nDataBytesPerChunk +
                                    tags.byteCount;
                                if (in->variantType == YAFFS_OBJECT_TYPE_FILE
                                    && in->variant.fileVariant.scannedFileSize <
@@ -4571,7 +4908,7 @@ static int yaffs_Scan(yaffs_Device * dev)
                                yaffs_SetChunkBit(dev, blk, c);
                                bi->pagesInUse++;
 
-                               yaffs_ReadChunkWithTagsFromNAND(dev, chunk,
+                               result = yaffs_ReadChunkWithTagsFromNAND(dev, chunk,
                                                                chunkData,
                                                                NULL);
 
@@ -4831,12 +5168,20 @@ static void yaffs_CheckObjectDetailsLoaded(yaffs_Object *in)
        __u8 *chunkData;
        yaffs_ObjectHeader *oh;
        yaffs_Device *dev = in->myDev;
+       yaffs_ExtendedTags tags;
+       int result;
        
+#if 0
+       T(YAFFS_TRACE_SCAN,(TSTR("details for object %d %s loaded" TENDSTR),
+               in->objectId,
+               in->lazyLoaded ? "not yet" : "already"));
+#endif
+               
        if(in->lazyLoaded){
                in->lazyLoaded = 0;
                chunkData = yaffs_GetTempBuffer(dev, __LINE__);
 
-               yaffs_ReadChunkWithTagsFromNAND(dev,in->chunkId,chunkData,NULL);
+               result = yaffs_ReadChunkWithTagsFromNAND(dev,in->chunkId,chunkData,&tags);
                oh = (yaffs_ObjectHeader *) chunkData;          
 
                in->yst_mode = oh->yst_mode;
@@ -4876,6 +5221,7 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
        int nBlocksToScan = 0;
 
        int chunk;
+       int result;
        int c;
        int deleted;
        yaffs_BlockState state;
@@ -4891,10 +5237,12 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
        
        int fileSize;
        int isShrink;
+       int foundChunksInBlock;
        int equivalentObjectId;
        
 
        yaffs_BlockIndex *blockIndex = NULL;
+       int altBlockIndex = 0;
 
        if (!dev->isYaffs2) {
                T(YAFFS_TRACE_SCAN,
@@ -4907,11 +5255,23 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
           ("yaffs_ScanBackwards starts  intstartblk %d intendblk %d..."
            TENDSTR), dev->internalStartBlock, dev->internalEndBlock));
 
-       chunkData = yaffs_GetTempBuffer(dev, __LINE__);
 
        dev->sequenceNumber = YAFFS_LOWEST_SEQUENCE_NUMBER;
 
        blockIndex = YMALLOC(nBlocks * sizeof(yaffs_BlockIndex));
+       
+       if(!blockIndex) {
+               blockIndex = YMALLOC_ALT(nBlocks * sizeof(yaffs_BlockIndex));
+               altBlockIndex = 1;
+       }
+       
+       if(!blockIndex) {
+               T(YAFFS_TRACE_SCAN,
+                 (TSTR("yaffs_Scan() could not allocate block index!" TENDSTR)));
+               return YAFFS_FAIL;
+       }
+       
+       chunkData = yaffs_GetTempBuffer(dev, __LINE__);
 
        /* Scan all the blocks to determine their state */
        for (blk = dev->internalStartBlock; blk <= dev->internalEndBlock; blk++) {
@@ -4925,11 +5285,18 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                bi->blockState = state;
                bi->sequenceNumber = sequenceNumber;
 
+               if(bi->sequenceNumber == YAFFS_SEQUENCE_CHECKPOINT_DATA)
+                       bi->blockState = state = YAFFS_BLOCK_STATE_CHECKPOINT;
+                       
                T(YAFFS_TRACE_SCAN_DEBUG,
                  (TSTR("Block scanning block %d state %d seq %d" TENDSTR), blk,
                   state, sequenceNumber));
 
-               if (state == YAFFS_BLOCK_STATE_DEAD) {
+               
+               if(state == YAFFS_BLOCK_STATE_CHECKPOINT){
+                       /* todo .. fix free space ? */
+                       
+               } else if (state == YAFFS_BLOCK_STATE_DEAD) {
                        T(YAFFS_TRACE_BAD_BLOCKS,
                          (TSTR("block %d is bad" TENDSTR), blk));
                } else if (state == YAFFS_BLOCK_STATE_EMPTY) {
@@ -4963,11 +5330,23 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                }
        }
 
-       /*
-        * Sort the blocks
-        * Dungy old bubble sort for now...
-        */
+       T(YAFFS_TRACE_SCAN,
+       (TSTR("%d blocks to be sorted..." TENDSTR), nBlocksToScan));
+
+
+
+       YYIELD();
+
+       /* Sort the blocks */
+#ifndef CONFIG_YAFFS_USE_OWN_SORT
        {
+               /* Use qsort now. */
+               qsort(blockIndex, nBlocksToScan, sizeof(yaffs_BlockIndex), ybicmp);
+       }
+#else
+       {
+               /* Dungy old bubble sort... */
+               
                yaffs_BlockIndex temp;
                int i;
                int j;
@@ -4980,6 +5359,11 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                                        blockIndex[i] = temp;
                                }
        }
+#endif
+
+       YYIELD();
+
+       T(YAFFS_TRACE_SCAN, (TSTR("...done" TENDSTR)));
 
        /* Now scan the blocks looking at the data. */
        startIterator = 0;
@@ -4990,6 +5374,9 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
        /* For each block.... backwards */
        for (blockIterator = endIterator; blockIterator >= startIterator;
             blockIterator--) {
+               /* Cooperative multitasking! This loop can run for so
+                  long that watchdog timers expire. */
+               YYIELD();
 
                /* get the block to scan in the correct order */
                blk = blockIndex[blockIterator].block;
@@ -5000,6 +5387,7 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                deleted = 0;
 
                /* For each chunk in each block that needs scanning.... */
+               foundChunksInBlock = 0;
                for (c = dev->nChunksPerBlock - 1; c >= 0 &&
                     (state == YAFFS_BLOCK_STATE_NEEDS_SCANNING ||
                      state == YAFFS_BLOCK_STATE_ALLOCATING); c--) {
@@ -5008,60 +5396,73 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                         */
                        chunk = blk * dev->nChunksPerBlock + c;
 
-                       yaffs_ReadChunkWithTagsFromNAND(dev, chunk, NULL,
+                       result = yaffs_ReadChunkWithTagsFromNAND(dev, chunk, NULL,
                                                        &tags);
 
                        /* Let's have a good look at this chunk... */
 
                        if (!tags.chunkUsed) {
-                               // An unassigned chunk in the block
-                               // This means that either the block is empty or 
-                               // this is the one being allocated from
+                               /* An unassigned chunk in the block.
+                                * If there are used chunks after this one, then
+                                * it is a chunk that was skipped due to failing the erased
+                                * check. Just skip it so that it can be deleted.
+                                * But, more typically, We get here when this is an unallocated
+                                * chunk and his means that either the block is empty or 
+                                * this is the one being allocated from
+                                */
 
-                               if (c == 0) {
+                               if(foundChunksInBlock)
+                               {
+                                       /* This is a chunk that was skipped due to failing the erased check */
+                                       
+                               } else if (c == 0) {
                                        /* We're looking at the first chunk in the block so the block is unused */
                                        state = YAFFS_BLOCK_STATE_EMPTY;
                                        dev->nErasedBlocks++;
                                } else {
-                                       /* this is the block being allocated from */
-                                       if (state ==
-                                           YAFFS_BLOCK_STATE_NEEDS_SCANNING) {
-                                               T(YAFFS_TRACE_SCAN,
-                                                 (TSTR
-                                                  (" Allocating from %d %d"
-                                                   TENDSTR), blk, c));
+                                       if (state == YAFFS_BLOCK_STATE_NEEDS_SCANNING ||
+                                           state == YAFFS_BLOCK_STATE_ALLOCATING) {
+                                               if(dev->sequenceNumber == bi->sequenceNumber) {
+                                                       /* this is the block being allocated from */
+                                               
+                                                       T(YAFFS_TRACE_SCAN,
+                                                         (TSTR
+                                                          (" Allocating from %d %d"
+                                                           TENDSTR), blk, c));
+
+                                                       state = YAFFS_BLOCK_STATE_ALLOCATING;
+                                                       dev->allocationBlock = blk;
+                                                       dev->allocationPage = c;
+                                                       dev->allocationBlockFinder = blk;       
+                                               }
+                                               else {
+                                                       /* This is a partially written block that is not
+                                                        * the current allocation block. This block must have
+                                                        * had a write failure, so set up for retirement.
+                                                        */
+                                                 
+                                                        bi->needsRetiring = 1;
+                                                        bi->gcPrioritise = 1;
+                                                                                                        
+                                                        T(YAFFS_TRACE_ALWAYS,
+                                                        (TSTR("Partially written block %d being set for retirement" TENDSTR),
+                                                        blk));
+                                               }
+
                                        }
-                                       state = YAFFS_BLOCK_STATE_ALLOCATING;
-                                       dev->allocationBlock = blk;
-                                       dev->allocationPage = c;
-                                       dev->allocationBlockFinder = blk;       
-                                       /* Set it to here to encourage the allocator to 
-                                        *  go forth from here.
-                                        */
                                         
-                                       /* Yaffs2 sanity check:
-                                        * This should be the one with the highest sequence number
-                                        */
-                                       if (dev->isYaffs2
-                                           && (dev->sequenceNumber !=
-                                               bi->sequenceNumber)) {
-                                               T(YAFFS_TRACE_ALWAYS,
-                                                 (TSTR
-                                                  ("yaffs: Allocation block %d was not highest sequence "
-                                                   "id: block seq = %d, dev seq = %d"
-                                                   TENDSTR), blk,
-                                                  bi->sequenceNumber,
-                                                  dev->sequenceNumber));
-                                       }
                                }
 
                                dev->nFreeChunks++;
+                               
                        } else if (tags.chunkId > 0) {
                                /* chunkId > 0 so it is a data chunk... */
                                unsigned int endpos;
-
                                __u32 chunkBase =
-                                   (tags.chunkId - 1) * dev->nBytesPerChunk;
+                                   (tags.chunkId - 1) * dev->nDataBytesPerChunk;
+                                                               
+                               foundChunksInBlock = 1;
+
 
                                yaffs_SetChunkBit(dev, blk, c);
                                bi->pagesInUse++;
@@ -5082,7 +5483,7 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                                         */
                                        endpos =
                                            (tags.chunkId -
-                                            1) * dev->nBytesPerChunk +
+                                            1) * dev->nDataBytesPerChunk +
                                            tags.byteCount;
                                            
                                        if (!in->valid &&       /* have not got an object header yet */
@@ -5105,6 +5506,8 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                                /* chunkId == 0, so it is an ObjectHeader.
                                 * Thus, we read in the object header and make the object
                                 */
+                               foundChunksInBlock = 1;
+
                                yaffs_SetChunkBit(dev, blk, c);
                                bi->pagesInUse++;
 
@@ -5132,7 +5535,7 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
                                         * living with invalid data until needed.
                                         */
 
-                                       yaffs_ReadChunkWithTagsFromNAND(dev,
+                                       result = yaffs_ReadChunkWithTagsFromNAND(dev,
                                                                        chunk,
                                                                        chunkData,
                                                                        NULL);
@@ -5409,9 +5812,11 @@ static int yaffs_ScanBackwards(yaffs_Device * dev)
 
        }
 
-       if (blockIndex) {
+       if (altBlockIndex) 
+               YFREE_ALT(blockIndex);
+       else
                YFREE(blockIndex);
-       }
+       
        /* Ok, we've done all the scanning.
         * Fix up the hard link chains.
         * We should now have scanned all the objects, now it's time to add these 
@@ -5642,14 +6047,15 @@ int yaffs_GetObjectName(yaffs_Object * obj, YCHAR * name, int buffSize)
        }
 #endif
        else {
+               int result;
                __u8 *buffer = yaffs_GetTempBuffer(obj->myDev, __LINE__);
 
                yaffs_ObjectHeader *oh = (yaffs_ObjectHeader *) buffer;
 
-               memset(buffer, 0, obj->myDev->nBytesPerChunk);
+               memset(buffer, 0, obj->myDev->nDataBytesPerChunk);
 
                if (obj->chunkId >= 0) {
-                       yaffs_ReadChunkWithTagsFromNAND(obj->myDev,
+                       result = yaffs_ReadChunkWithTagsFromNAND(obj->myDev,
                                                        obj->chunkId, buffer,
                                                        NULL);
                }
@@ -5674,7 +6080,7 @@ int yaffs_GetObjectFileLength(yaffs_Object * obj)
                return yaffs_strlen(obj->variant.symLinkVariant.alias);
        } else {
                /* Only a directory should drop through to here */
-               return obj->myDev->nBytesPerChunk;
+               return obj->myDev->nDataBytesPerChunk;
        }
 }
 
@@ -5876,7 +6282,6 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
 {
        unsigned x;
        int bits;
-       int extraBits;
 
        T(YAFFS_TRACE_TRACING, (TSTR("yaffs: yaffs_GutsInitialise()" TENDSTR)));
 
@@ -5902,8 +6307,8 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
 
        /* Check geometry parameters. */
 
-       if ((dev->isYaffs2 && dev->nBytesPerChunk < 1024) || 
-           (!dev->isYaffs2 && dev->nBytesPerChunk != 512) || 
+       if ((dev->isYaffs2 && dev->nDataBytesPerChunk < 1024) || 
+           (!dev->isYaffs2 && dev->nDataBytesPerChunk != 512) || 
             dev->nChunksPerBlock < 2 || 
             dev->nReservedBlocks < 2 || 
             dev->internalStartBlock <= 0 || 
@@ -5913,7 +6318,7 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
                T(YAFFS_TRACE_ALWAYS,
                  (TSTR
                   ("yaffs: NAND geometry problems: chunk size %d, type is yaffs%s "
-                   TENDSTR), dev->nBytesPerChunk, dev->isYaffs2 ? "2" : ""));
+                   TENDSTR), dev->nDataBytesPerChunk, dev->isYaffs2 ? "2" : ""));
                return YAFFS_FAIL;
        }
 
@@ -5952,21 +6357,38 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
 
 
 
-       /* OK now calculate a few things for the device
+       /* OK now calculate a few things for the device */
+       
+       /*
+        *  Calculate all the chunk size manipulation numbers: 
+        */
+        /* Start off assuming it is a power of 2 */
+        dev->chunkShift = ShiftDiv(dev->nDataBytesPerChunk);
+        dev->chunkMask = (1<<dev->chunkShift) - 1;
+
+        if(dev->nDataBytesPerChunk == (dev->chunkMask + 1)){
+               /* Yes it is a power of 2, disable crumbs */
+               dev->crumbMask = 0;
+               dev->crumbShift = 0;
+               dev->crumbsPerChunk = 0;
+        } else {
+               /* Not a power of 2, use crumbs instead */
+               dev->crumbShift = ShiftDiv(sizeof(yaffs_PackedTags2TagsPart));
+               dev->crumbMask = (1<<dev->crumbShift)-1;
+               dev->crumbsPerChunk = dev->nDataBytesPerChunk/(1 << dev->crumbShift);
+               dev->chunkShift = 0;
+               dev->chunkMask = 0;
+       }
+               
+
+       /*
         * Calculate chunkGroupBits.
         * We need to find the next power of 2 > than internalEndBlock
         */
 
        x = dev->nChunksPerBlock * (dev->internalEndBlock + 1);
-
-       for (bits = extraBits = 0; x > 1; bits++) {
-               if (x & 1)
-                       extraBits++;
-               x >>= 1;
-       }
-
-       if (extraBits > 0)
-               bits++;
+       
+       bits = ShiftsGE(x);
        
        /* Set up tnode width if wide tnodes are enabled. */
        if(!dev->wideTnodesDisabled){
@@ -6026,6 +6448,7 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
        dev->nErasureFailures = 0;
        dev->nErasedBlocks = 0;
        dev->isDoingGC = 0;
+       dev->hasPendingPrioritisedGCs = 1; /* Assume the worst for now, will get fixed on first GC */
 
        /* Initialise temporary buffers and caches. */
        {
@@ -6033,7 +6456,7 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
                for (i = 0; i < YAFFS_N_TEMP_BUFFERS; i++) {
                        dev->tempBuffer[i].line = 0;    /* not in use */
                        dev->tempBuffer[i].buffer =
-                           YMALLOC(dev->nBytesPerChunk);
+                           YMALLOC_DMA(dev->nDataBytesPerChunk);
                }
        }
        
@@ -6051,7 +6474,7 @@ int yaffs_GutsInitialise(yaffs_Device * dev)
                        dev->srCache[i].object = NULL;
                        dev->srCache[i].lastUse = 0;
                        dev->srCache[i].dirty = 0;
-                       dev->srCache[i].data = YMALLOC(dev->nBytesPerChunk);
+                       dev->srCache[i].data = YMALLOC_DMA(dev->nDataBytesPerChunk);
                }
                dev->srLastUse = 0;
        }
@@ -6174,6 +6597,7 @@ int yaffs_GetNumberOfFreeChunks(yaffs_Device * dev)
 
        int nFree;
        int nDirtyCacheChunks;
+       int blocksForCheckpoint;
 
 #if 1
        nFree = dev->nFreeChunks;
@@ -6196,6 +6620,13 @@ int yaffs_GetNumberOfFreeChunks(yaffs_Device * dev)
        nFree -= nDirtyCacheChunks;
 
        nFree -= ((dev->nReservedBlocks + 1) * dev->nChunksPerBlock);
+       
+       /* Now we figure out how much to reserve for the checkpoint and report that... */
+       blocksForCheckpoint = dev->nCheckpointReservedBlocks - dev->blocksInCheckpoint;
+       if(blocksForCheckpoint < 0)
+               blocksForCheckpoint = 0;
+               
+       nFree -= (blocksForCheckpoint * dev->nChunksPerBlock);
 
        if (nFree < 0)
                nFree = 0;