Update clang to r96341.

14 files changed, 1069 insertions, 253 deletions

diff --git a/tools/CIndex/CIndex.cpp b/tools/CIndex/CIndex.cpp
index 84f908d57183..f9995eb743df 100644
--- a/tools/CIndex/CIndex.cpp
+++ b/tools/CIndex/CIndex.cpp
@@ -14,11 +14,15 @@
 
 #include "CIndexer.h"
 #include "CXCursor.h"
+#include "CXSourceLocation.h"
+#include "CIndexDiagnostic.h"
 
 #include "clang/Basic/Version.h"
+
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/TypeLocVisitor.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -116,34 +120,6 @@ public:
 #endif
 #endif
 
-typedef llvm::PointerIntPair<ASTContext *, 1, bool> CXSourceLocationPtr;
-
-/// \brief Translate a Clang source location into a CIndex source location.
-static CXSourceLocation translateSourceLocation(ASTContext &Context,
-                                                SourceLocation Loc,
-                                                bool AtEnd = false) {
-  CXSourceLocationPtr Ptr(&Context, AtEnd);
-  CXSourceLocation Result = { Ptr.getOpaqueValue(), Loc.getRawEncoding() };
-  return Result;
-}
-
-/// \brief Translate a Clang source range into a CIndex source range.
-static CXSourceRange translateSourceRange(ASTContext &Context, SourceRange R) {
-  CXSourceRange Result = { &Context, 
-                           R.getBegin().getRawEncoding(),
-                           R.getEnd().getRawEncoding() };
-  return Result;
-}
-
-static SourceLocation translateSourceLocation(CXSourceLocation L) {
-  return SourceLocation::getFromRawEncoding(L.int_data);
-}
-
-static SourceRange translateSourceRange(CXSourceRange R) {
-  return SourceRange(SourceLocation::getFromRawEncoding(R.begin_int_data),
-                     SourceLocation::getFromRawEncoding(R.end_int_data));
-}
-
 /// \brief The result of comparing two source ranges.
 enum RangeComparisonResult {
   /// \brief Either the ranges overlap or one of the ranges is invalid.
@@ -163,13 +139,57 @@ static RangeComparisonResult RangeCompare(SourceManager &SM,
                                           SourceRange R2) {
   assert(R1.isValid() && "First range is invalid?");
   assert(R2.isValid() && "Second range is invalid?");
-  if (SM.isBeforeInTranslationUnit(R1.getEnd(), R2.getBegin()))
+  if (R1.getEnd() == R2.getBegin() ||
+      SM.isBeforeInTranslationUnit(R1.getEnd(), R2.getBegin()))
     return RangeBefore;
-  if (SM.isBeforeInTranslationUnit(R2.getEnd(), R1.getBegin()))
+  if (R2.getEnd() == R1.getBegin() ||
+      SM.isBeforeInTranslationUnit(R2.getEnd(), R1.getBegin()))
     return RangeAfter;
   return RangeOverlap;
 }
 
+/// \brief Translate a Clang source range into a CIndex source range.
+///
+/// Clang internally represents ranges where the end location points to the
+/// start of the token at the end. However, for external clients it is more
+/// useful to have a CXSourceRange be a proper half-open interval. This routine
+/// does the appropriate translation.
+CXSourceRange cxloc::translateSourceRange(const SourceManager &SM, 
+                                          const LangOptions &LangOpts,
+                                          SourceRange R) {
+  // FIXME: This is largely copy-paste from
+  // TextDiagnosticPrinter::HighlightRange.  When it is clear that this is what
+  // we want the two routines should be refactored.
+
+  // We want the last character in this location, so we will adjust the
+  // instantiation location accordingly.
+
+  // If the location is from a macro instantiation, get the end of the
+  // instantiation range.
+  SourceLocation EndLoc = R.getEnd();
+  SourceLocation InstLoc = SM.getInstantiationLoc(EndLoc);
+  if (EndLoc.isMacroID())
+    InstLoc = SM.getInstantiationRange(EndLoc).second;
+
+  // Measure the length token we're pointing at, so we can adjust the physical
+  // location in the file to point at the last character.
+  //
+  // FIXME: This won't cope with trigraphs or escaped newlines well. For that,
+  // we actually need a preprocessor, which isn't currently available
+  // here. Eventually, we'll switch the pointer data of
+  // CXSourceLocation/CXSourceRange to a translation unit (CXXUnit), so that the
+  // preprocessor will be available here. At that point, we can use
+  // Preprocessor::getLocForEndOfToken().
+  if (InstLoc.isValid()) {
+    unsigned Length = Lexer::MeasureTokenLength(InstLoc, SM, LangOpts);
+    EndLoc = EndLoc.getFileLocWithOffset(Length);
+  }
+
+  CXSourceRange Result = { { (void *)&SM, (void *)&LangOpts },
+                           R.getBegin().getRawEncoding(),
+                           EndLoc.getRawEncoding() };
+  return Result;
+}
 
 //===----------------------------------------------------------------------===//
 // Cursor visitor.
@@ -214,15 +234,9 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   /// \brief Determine whether this particular source range comes before, comes 
   /// after, or overlaps the region of interest. 
   ///
-  /// \param R a source range retrieved from the abstract syntax tree.
+  /// \param R a half-open source range retrieved from the abstract syntax tree.
   RangeComparisonResult CompareRegionOfInterest(SourceRange R); 
   
-  /// \brief Determine whether this particular source range comes before, comes 
-  /// after, or overlaps the region of interest. 
-  ///
-  /// \param CXR a source range retrieved from a cursor.
-  RangeComparisonResult CompareRegionOfInterest(CXSourceRange CXR); 
-  
 public:
   CursorVisitor(ASTUnit *TU, CXCursorVisitor Visitor, CXClientData ClientData, 
                 unsigned MaxPCHLevel, 
@@ -292,6 +306,8 @@ public:
   // FIXME: LabelStmt label?
   bool VisitIfStmt(IfStmt *S);
   bool VisitSwitchStmt(SwitchStmt *S);
+  bool VisitWhileStmt(WhileStmt *S);
+  bool VisitForStmt(ForStmt *S);
   
   // Expression visitors
   bool VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E);
@@ -302,20 +318,9 @@ public:
 } // end anonymous namespace
 
 RangeComparisonResult CursorVisitor::CompareRegionOfInterest(SourceRange R) {
-  assert(RegionOfInterest.isValid() && "RangeCompare called with invalid range");
-  if (R.isInvalid())
-    return RangeOverlap;
-
-  // Move the end of the input range to the end of the last token in that
-  // range.
-  R.setEnd(TU->getPreprocessor().getLocForEndOfToken(R.getEnd(), 1));
   return RangeCompare(TU->getSourceManager(), R, RegionOfInterest);
 }
 
-RangeComparisonResult CursorVisitor::CompareRegionOfInterest(CXSourceRange CXR) {
-  return CompareRegionOfInterest(translateSourceRange(CXR));
-}
-
 /// \brief Visit the given cursor and, if requested by the visitor,
 /// its children.
 ///
@@ -343,9 +348,9 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) {
   // If we have a range of interest, and this cursor doesn't intersect with it,
   // we're done.
   if (RegionOfInterest.isValid() && !CheckedRegionOfInterest) {
-    CXSourceRange Range = clang_getCursorExtent(Cursor);
-    if (translateSourceRange(Range).isInvalid() || 
-        CompareRegionOfInterest(Range))
+    SourceRange Range =
+      cxloc::translateCXSourceRange(clang_getCursorExtent(Cursor));
+    if (Range.isInvalid() || CompareRegionOfInterest(Range))
       return false;
   }
   
@@ -360,7 +365,7 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) {
     return VisitChildren(Cursor);
   }
 
-  llvm_unreachable("Silly GCC, we can't get here");
+  return false;
 }
 
 /// \brief Visit the children of the given cursor.
@@ -432,12 +437,15 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
 bool CursorVisitor::VisitDeclContext(DeclContext *DC) {
   for (DeclContext::decl_iterator
        I = DC->decls_begin(), E = DC->decls_end(); I != E; ++I) {
+    CXCursor Cursor = MakeCXCursor(*I, TU);
+
     if (RegionOfInterest.isValid()) {
-      SourceRange R = (*I)->getSourceRange();
-      if (R.isInvalid())
+      SourceRange Range =
+        cxloc::translateCXSourceRange(clang_getCursorExtent(Cursor));
+      if (Range.isInvalid())
         continue;
-      
-      switch (CompareRegionOfInterest(R)) {
+
+      switch (CompareRegionOfInterest(Range)) {
       case RangeBefore:
         // This declaration comes before the region of interest; skip it.
         continue;
@@ -452,7 +460,7 @@ bool CursorVisitor::VisitDeclContext(DeclContext *DC) {
       }      
     }
     
-    if (Visit(MakeCXCursor(*I, TU), true))
+    if (Visit(Cursor, true))
       return true;
   }
   
@@ -783,7 +791,7 @@ bool CursorVisitor::VisitTypeOfTypeLoc(TypeOfTypeLoc TL) {
 bool CursorVisitor::VisitStmt(Stmt *S) {
   for (Stmt::child_iterator Child = S->child_begin(), ChildEnd = S->child_end();
        Child != ChildEnd; ++Child) {
-    if (Visit(MakeCXCursor(*Child, StmtParent, TU)))
+    if (*Child && Visit(MakeCXCursor(*Child, StmtParent, TU)))
       return true;
   }
   
@@ -793,7 +801,7 @@ bool CursorVisitor::VisitStmt(Stmt *S) {
 bool CursorVisitor::VisitDeclStmt(DeclStmt *S) {
   for (DeclStmt::decl_iterator D = S->decl_begin(), DEnd = S->decl_end();
        D != DEnd; ++D) {
-    if (Visit(MakeCXCursor(*D, TU)))
+    if (*D && Visit(MakeCXCursor(*D, TU)))
       return true;
   }
   
@@ -804,12 +812,12 @@ bool CursorVisitor::VisitIfStmt(IfStmt *S) {
   if (VarDecl *Var = S->getConditionVariable()) {
     if (Visit(MakeCXCursor(Var, TU)))
       return true;
-  } else if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
-    return true;
+  } 
   
+  if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
+    return true;
   if (S->getThen() && Visit(MakeCXCursor(S->getThen(), StmtParent, TU)))
     return true;
-
   if (S->getElse() && Visit(MakeCXCursor(S->getElse(), StmtParent, TU)))
     return true;
 
@@ -820,9 +828,42 @@ bool CursorVisitor::VisitSwitchStmt(SwitchStmt *S) {
   if (VarDecl *Var = S->getConditionVariable()) {
     if (Visit(MakeCXCursor(Var, TU)))
       return true;
-  } else if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
+  } 
+  
+  if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
+    return true;
+  if (S->getBody() && Visit(MakeCXCursor(S->getBody(), StmtParent, TU)))
+    return true;
+  
+  return false;
+}
+
+bool CursorVisitor::VisitWhileStmt(WhileStmt *S) {
+  if (VarDecl *Var = S->getConditionVariable()) {
+    if (Visit(MakeCXCursor(Var, TU)))
+      return true;
+  } 
+  
+  if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
+    return true;
+  if (S->getBody() && Visit(MakeCXCursor(S->getBody(), StmtParent, TU)))
     return true;
 
+  return false;
+}
+
+bool CursorVisitor::VisitForStmt(ForStmt *S) {
+  if (S->getInit() && Visit(MakeCXCursor(S->getInit(), StmtParent, TU)))
+    return true;
+  if (VarDecl *Var = S->getConditionVariable()) {
+    if (Visit(MakeCXCursor(Var, TU)))
+      return true;
+  } 
+  
+  if (S->getCond() && Visit(MakeCXCursor(S->getCond(), StmtParent, TU)))
+    return true;
+  if (S->getInc() && Visit(MakeCXCursor(S->getInc(), StmtParent, TU)))
+    return true;
   if (S->getBody() && Visit(MakeCXCursor(S->getBody(), StmtParent, TU)))
     return true;
   
@@ -868,37 +909,59 @@ CXString CIndexer::createCXString(const char *String, bool DupString){
   return Str;
 }
 
+CXString CIndexer::createCXString(llvm::StringRef String, bool DupString) {
+  CXString Result;
+  if (DupString || (!String.empty() && String.data()[String.size()] != 0)) {
+    char *Spelling = (char *)malloc(String.size() + 1);
+    memmove(Spelling, String.data(), String.size());
+    Spelling[String.size()] = 0;
+    Result.Spelling = Spelling;
+    Result.MustFreeString = 1;
+  } else {
+    Result.Spelling = String.data();
+    Result.MustFreeString = 0;
+  }
+  return Result;
+}
+
 extern "C" {
-CXIndex clang_createIndex(int excludeDeclarationsFromPCH,
-                          int displayDiagnostics) {
+CXIndex clang_createIndex(int excludeDeclarationsFromPCH) {
   CIndexer *CIdxr = new CIndexer();
   if (excludeDeclarationsFromPCH)
     CIdxr->setOnlyLocalDecls();
-  if (displayDiagnostics)
-    CIdxr->setDisplayDiagnostics();
   return CIdxr;
 }
 
 void clang_disposeIndex(CXIndex CIdx) {
-  assert(CIdx && "Passed null CXIndex");
-  delete static_cast<CIndexer *>(CIdx);
+  if (CIdx)
+    delete static_cast<CIndexer *>(CIdx);
 }
 
 void clang_setUseExternalASTGeneration(CXIndex CIdx, int value) {
-  assert(CIdx && "Passed null CXIndex");
-  CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
-  CXXIdx->setUseExternalASTGeneration(value);
+  if (CIdx) {
+    CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
+    CXXIdx->setUseExternalASTGeneration(value);
+  }
 }
 
-// FIXME: need to pass back error info.
 CXTranslationUnit clang_createTranslationUnit(CXIndex CIdx,
-                                              const char *ast_filename) {
-  assert(CIdx && "Passed null CXIndex");
+                                              const char *ast_filename,
+                                             CXDiagnosticCallback diag_callback,
+                                              CXClientData diag_client_data) {
+  if (!CIdx)
+    return 0;
+  
   CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
 
-  return ASTUnit::LoadFromPCHFile(ast_filename, CXXIdx->getDiags(),
-                                  CXXIdx->getOnlyLocalDecls(),
-                                  /* UseBumpAllocator = */ true);
+  // Configure the diagnostics.
+  DiagnosticOptions DiagOpts;
+  llvm::OwningPtr<Diagnostic> Diags;
+  Diags.reset(CompilerInstance::createDiagnostics(DiagOpts, 0, 0));
+  CIndexDiagnosticClient DiagClient(diag_callback, diag_client_data);
+  Diags->setClient(&DiagClient);
+  
+  return ASTUnit::LoadFromPCHFile(ast_filename, *Diags,
+                                  CXXIdx->getOnlyLocalDecls());
 }
 
 CXTranslationUnit
@@ -907,10 +970,21 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
                                           int num_command_line_args,
                                           const char **command_line_args,
                                           unsigned num_unsaved_files,
-                                          struct CXUnsavedFile *unsaved_files) {
-  assert(CIdx && "Passed null CXIndex");
+                                          struct CXUnsavedFile *unsaved_files,
+                                          CXDiagnosticCallback diag_callback,
+                                          CXClientData diag_client_data) {
+  if (!CIdx)
+    return 0;
+  
   CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
 
+  // Configure the diagnostics.
+  DiagnosticOptions DiagOpts;
+  llvm::OwningPtr<Diagnostic> Diags;
+  Diags.reset(CompilerInstance::createDiagnostics(DiagOpts, 0, 0));
+  CIndexDiagnosticClient DiagClient(diag_callback, diag_client_data);
+  Diags->setClient(&DiagClient);
+                               
   llvm::SmallVector<ASTUnit::RemappedFile, 4> RemappedFiles;
   for (unsigned I = 0; I != num_unsaved_files; ++I) {
     const llvm::MemoryBuffer *Buffer 
@@ -932,7 +1006,7 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
     Args.insert(Args.end(), command_line_args,
                 command_line_args + num_command_line_args);
 
-    unsigned NumErrors = CXXIdx->getDiags().getNumErrors();
+    unsigned NumErrors = Diags->getNumErrors();
     
 #ifdef USE_CRASHTRACER
     ArgsCrashTracerInfo ACTI(Args);
@@ -940,16 +1014,15 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
     
     llvm::OwningPtr<ASTUnit> Unit(
       ASTUnit::LoadFromCommandLine(Args.data(), Args.data() + Args.size(),
-                                   CXXIdx->getDiags(),
+                                   *Diags, 
                                    CXXIdx->getClangResourcesPath(),
                                    CXXIdx->getOnlyLocalDecls(),
-                                   /* UseBumpAllocator = */ true,
                                    RemappedFiles.data(),
                                    RemappedFiles.size()));
     
     // FIXME: Until we have broader testing, just drop the entire AST if we
     // encountered an error.
-    if (NumErrors != CXXIdx->getDiags().getNumErrors())
+    if (NumErrors != Diags->getNumErrors())
       return 0;
 
     return Unit.take();
@@ -1004,6 +1077,13 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
       argv.push_back(arg);
     }
 
+  // Generate a temporary name for the diagnostics file.
+  char tmpFileResults[L_tmpnam];
+  char *tmpResultsFileName = tmpnam(tmpFileResults);
+  llvm::sys::Path DiagnosticsFile(tmpResultsFileName);
+  TemporaryFiles.push_back(DiagnosticsFile);
+  argv.push_back("-fdiagnostics-binary");
+
   // Add the null terminator.
   argv.push_back(NULL);
 
@@ -1011,30 +1091,39 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
   llvm::sys::Path DevNull; // leave empty, causes redirection to /dev/null
                            // on Unix or NUL (Windows).
   std::string ErrMsg;
-  const llvm::sys::Path *Redirects[] = { &DevNull, &DevNull, &DevNull, NULL };
+  const llvm::sys::Path *Redirects[] = { &DevNull, &DevNull, &DiagnosticsFile,
+                                         NULL };
   llvm::sys::Program::ExecuteAndWait(ClangPath, &argv[0], /* env */ NULL,
-      /* redirects */ !CXXIdx->getDisplayDiagnostics() ? &Redirects[0] : NULL,
+      /* redirects */ &Redirects[0],
       /* secondsToWait */ 0, /* memoryLimits */ 0, &ErrMsg);
 
-  if (CXXIdx->getDisplayDiagnostics() && !ErrMsg.empty()) {
-    llvm::errs() << "clang_createTranslationUnitFromSourceFile: " << ErrMsg
-                 << '\n' << "Arguments: \n";
+  if (!ErrMsg.empty()) {
+    std::string AllArgs;
     for (std::vector<const char*>::iterator I = argv.begin(), E = argv.end();
-         I!=E; ++I) {
+         I != E; ++I) {
+      AllArgs += ' ';
       if (*I)
-        llvm::errs() << ' ' << *I << '\n';
+        AllArgs += *I;
     }
-    llvm::errs() << '\n';
+    
+    Diags->Report(diag::err_fe_clang) << AllArgs << ErrMsg;
   }
 
-  ASTUnit *ATU = ASTUnit::LoadFromPCHFile(astTmpFile, CXXIdx->getDiags(),
+  // FIXME: Parse the (redirected) standard error to emit diagnostics.
+
+  ASTUnit *ATU = ASTUnit::LoadFromPCHFile(astTmpFile, *Diags,
                                           CXXIdx->getOnlyLocalDecls(),
-                                          /* UseBumpAllocator = */ true,
                                           RemappedFiles.data(),
                                           RemappedFiles.size());
   if (ATU)
     ATU->unlinkTemporaryFile();
   
+  // FIXME: Currently we don't report diagnostics on invalid ASTs.
+  if (ATU)
+    ReportSerializedDiagnostics(DiagnosticsFile, *Diags, 
+                                num_unsaved_files, unsaved_files,
+                                ATU->getASTContext().getLangOptions());
+
   for (unsigned i = 0, e = TemporaryFiles.size(); i != e; ++i)
     TemporaryFiles[i].eraseFromDisk();
   
@@ -1042,12 +1131,14 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
 }
 
 void clang_disposeTranslationUnit(CXTranslationUnit CTUnit) {
-  assert(CTUnit && "Passed null CXTranslationUnit");
-  delete static_cast<ASTUnit *>(CTUnit);
+  if (CTUnit)
+    delete static_cast<ASTUnit *>(CTUnit);
 }
 
 CXString clang_getTranslationUnitSpelling(CXTranslationUnit CTUnit) {
-  assert(CTUnit && "Passed null CXTranslationUnit");
+  if (!CTUnit)
+    return CIndexer::createCXString("");
+  
   ASTUnit *CXXUnit = static_cast<ASTUnit *>(CTUnit);
   return CIndexer::createCXString(CXXUnit->getOriginalSourceFileName().c_str(),
                                   true);
@@ -1066,12 +1157,14 @@ CXCursor clang_getTranslationUnitCursor(CXTranslationUnit TU) {
 
 extern "C" {
 CXSourceLocation clang_getNullLocation() {
-  CXSourceLocation Result = { 0, 0 };
+  CXSourceLocation Result = { { 0, 0 }, 0 };
   return Result;
 }
 
 unsigned clang_equalLocations(CXSourceLocation loc1, CXSourceLocation loc2) {
-  return loc1.ptr_data == loc2.ptr_data && loc1.int_data == loc2.int_data;
+  return (loc1.ptr_data[0] == loc2.ptr_data[0] &&
+          loc1.ptr_data[1] == loc2.ptr_data[1] &&
+          loc1.int_data == loc2.int_data);
 }
 
 CXSourceLocation clang_getLocation(CXTranslationUnit tu,
@@ -1087,67 +1180,46 @@ CXSourceLocation clang_getLocation(CXTranslationUnit tu,
                                         static_cast<const FileEntry *>(file), 
                                               line, column);
   
-  return translateSourceLocation(CXXUnit->getASTContext(), SLoc, false);
+  return cxloc::translateSourceLocation(CXXUnit->getASTContext(), SLoc);
+}
+
+CXSourceRange clang_getNullRange() {
+  CXSourceRange Result = { { 0, 0 }, 0, 0 };
+  return Result;
 }
 
 CXSourceRange clang_getRange(CXSourceLocation begin, CXSourceLocation end) {
-  if (begin.ptr_data != end.ptr_data) {
-    CXSourceRange Result = { 0, 0, 0 };
-    return Result;
-  }
+  if (begin.ptr_data[0] != end.ptr_data[0] ||
+      begin.ptr_data[1] != end.ptr_data[1])
+    return clang_getNullRange();
   
-  CXSourceRange Result = { begin.ptr_data, begin.int_data, end.int_data };
+  CXSourceRange Result = { { begin.ptr_data[0], begin.ptr_data[1] }, 
+                           begin.int_data, end.int_data };
   return Result;
 }
 
 void clang_getInstantiationLocation(CXSourceLocation location,
                                     CXFile *file,
                                     unsigned *line,
-                                    unsigned *column) {
-  CXSourceLocationPtr Ptr
-    = CXSourceLocationPtr::getFromOpaqueValue(location.ptr_data);
+                                    unsigned *column,
+                                    unsigned *offset) {
   SourceLocation Loc = SourceLocation::getFromRawEncoding(location.int_data);
 
-  if (!Ptr.getPointer() || Loc.isInvalid()) {
+  if (!location.ptr_data[0] || Loc.isInvalid()) {
     if (file)
       *file = 0;
     if (line)
       *line = 0;
     if (column)
       *column = 0;
+    if (offset)
+      *offset = 0;
     return;
   }
 
-  // FIXME: This is largely copy-paste from
-  ///TextDiagnosticPrinter::HighlightRange.  When it is clear that this is
-  // what we want the two routines should be refactored.  
-  ASTContext &Context = *Ptr.getPointer();
-  SourceManager &SM = Context.getSourceManager();
+  const SourceManager &SM =
+    *static_cast<const SourceManager*>(location.ptr_data[0]);
   SourceLocation InstLoc = SM.getInstantiationLoc(Loc);
-  
-  if (Ptr.getInt()) {
-    // We want the last character in this location, so we will adjust
-    // the instantiation location accordingly.
-
-    // If the location is from a macro instantiation, get the end of
-    // the instantiation range.
-    if (Loc.isMacroID())
-      InstLoc = SM.getInstantiationRange(Loc).second;
-
-    // Measure the length token we're pointing at, so we can adjust
-    // the physical location in the file to point at the last
-    // character.
-    // FIXME: This won't cope with trigraphs or escaped newlines
-    // well. For that, we actually need a preprocessor, which isn't
-    // currently available here. Eventually, we'll switch the pointer
-    // data of CXSourceLocation/CXSourceRange to a translation unit
-    // (CXXUnit), so that the preprocessor will be available here. At
-    // that point, we can use Preprocessor::getLocForEndOfToken().
-    unsigned Length = Lexer::MeasureTokenLength(InstLoc, SM, 
-                                                Context.getLangOptions());
-    if (Length > 0)
-      InstLoc = InstLoc.getFileLocWithOffset(Length - 1);
-  }
 
   if (file)
     *file = (void *)SM.getFileEntryForID(SM.getFileID(InstLoc));
@@ -1155,18 +1227,19 @@ void clang_getInstantiationLocation(CXSourceLocation location,
     *line = SM.getInstantiationLineNumber(InstLoc);
   if (column)
     *column = SM.getInstantiationColumnNumber(InstLoc);
+  if (offset)
+    *offset = SM.getDecomposedLoc(InstLoc).second;
 }
 
 CXSourceLocation clang_getRangeStart(CXSourceRange range) {
-  CXSourceLocation Result = { range.ptr_data, range.begin_int_data };
+  CXSourceLocation Result = { { range.ptr_data[0], range.ptr_data[1] }, 
+                              range.begin_int_data };
   return Result;
 }
 
 CXSourceLocation clang_getRangeEnd(CXSourceRange range) {
-  llvm::PointerIntPair<ASTContext *, 1, bool> Ptr;
-  Ptr.setPointer(static_cast<ASTContext *>(range.ptr_data));
-  Ptr.setInt(true);
-  CXSourceLocation Result = { Ptr.getOpaqueValue(), range.end_int_data };
+  CXSourceLocation Result = { { range.ptr_data[0], range.ptr_data[1] },
+                              range.end_int_data };
   return Result;
 }
 
@@ -1181,7 +1254,6 @@ const char *clang_getFileName(CXFile SFile) {
   if (!SFile)
     return 0;
   
-  assert(SFile && "Passed null CXFile");
   FileEntry *FEnt = static_cast<FileEntry *>(SFile);
   return FEnt->getName();
 }
@@ -1190,7 +1262,6 @@ time_t clang_getFileTime(CXFile SFile) {
   if (!SFile)
     return 0;
   
-  assert(SFile && "Passed null CXFile");
   FileEntry *FEnt = static_cast<FileEntry *>(SFile);
   return FEnt->getModificationTime();
 }
@@ -1230,6 +1301,18 @@ static Decl *getDeclFromExpr(Stmt *E) {
   return 0;
 }
 
+static SourceLocation getLocationFromExpr(Expr *E) {
+  if (ObjCMessageExpr *Msg = dyn_cast<ObjCMessageExpr>(E))
+    return /*FIXME:*/Msg->getLeftLoc();
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
+    return DRE->getLocation();
+  if (MemberExpr *Member = dyn_cast<MemberExpr>(E))
+    return Member->getMemberLoc();
+  if (ObjCIvarRefExpr *Ivar = dyn_cast<ObjCIvarRefExpr>(E))
+    return Ivar->getLocation();
+  return E->getLocStart();
+}
+
 extern "C" {
   
 unsigned clang_visitChildren(CXCursor parent, 
@@ -1274,7 +1357,6 @@ static CXString getDeclSpelling(Decl *D) {
 }
     
 CXString clang_getCursorSpelling(CXCursor C) {
-  assert(getCursorDecl(C) && "CXCursor has null decl");
   if (clang_isTranslationUnit(C.kind))
     return clang_getTranslationUnitSpelling(C.data[2]);
 
@@ -1314,7 +1396,10 @@ CXString clang_getCursorSpelling(CXCursor C) {
     return CIndexer::createCXString("");
   }
 
-  return getDeclSpelling(getCursorDecl(C));
+  if (clang_isDeclaration(C.kind))
+    return getDeclSpelling(getCursorDecl(C));
+  
+  return CIndexer::createCXString("");
 }
 
 const char *clang_getCursorKindSpelling(enum CXCursorKind Kind) {
@@ -1373,11 +1458,10 @@ CXCursor clang_getCursor(CXTranslationUnit TU, CXSourceLocation Loc) {
   
   ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
 
-  SourceLocation SLoc = translateSourceLocation(Loc);
+  SourceLocation SLoc = cxloc::translateSourceLocation(Loc);
   CXCursor Result = MakeCXCursorInvalid(CXCursor_NoDeclFound);
   if (SLoc.isValid()) {
-    SourceRange RegionOfInterest(SLoc, 
-                       CXXUnit->getPreprocessor().getLocForEndOfToken(SLoc, 1));
+    SourceRange RegionOfInterest(SLoc, SLoc.getFileLocWithOffset(1));
     
     // FIXME: Would be great to have a "hint" cursor, then walk from that
     // hint cursor upward until we find a cursor whose source range encloses
@@ -1426,42 +1510,30 @@ CXCursorKind clang_getCursorKind(CXCursor C) {
   return C.kind;
 }
 
-static SourceLocation getLocationFromExpr(Expr *E) {
-  if (ObjCMessageExpr *Msg = dyn_cast<ObjCMessageExpr>(E))
-    return /*FIXME:*/Msg->getLeftLoc();
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-    return DRE->getLocation();
-  if (MemberExpr *Member = dyn_cast<MemberExpr>(E))
-    return Member->getMemberLoc();
-  if (ObjCIvarRefExpr *Ivar = dyn_cast<ObjCIvarRefExpr>(E))
-    return Ivar->getLocation();
-  return E->getLocStart();
-}
-
 CXSourceLocation clang_getCursorLocation(CXCursor C) {
   if (clang_isReference(C.kind)) {
     switch (C.kind) {
     case CXCursor_ObjCSuperClassRef: {       
       std::pair<ObjCInterfaceDecl *, SourceLocation> P
         = getCursorObjCSuperClassRef(C);
-      return translateSourceLocation(P.first->getASTContext(), P.second);
+      return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
     case CXCursor_ObjCProtocolRef: {       
       std::pair<ObjCProtocolDecl *, SourceLocation> P
         = getCursorObjCProtocolRef(C);
-      return translateSourceLocation(P.first->getASTContext(), P.second);
+      return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
     case CXCursor_ObjCClassRef: {       
       std::pair<ObjCInterfaceDecl *, SourceLocation> P
         = getCursorObjCClassRef(C);
-      return translateSourceLocation(P.first->getASTContext(), P.second);
+      return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
     case CXCursor_TypeRef: {       
       std::pair<TypeDecl *, SourceLocation> P = getCursorTypeRef(C);
-      return translateSourceLocation(P.first->getASTContext(), P.second);
+      return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
       
     default:
@@ -1471,19 +1543,17 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
   }
 
   if (clang_isExpression(C.kind))
-    return translateSourceLocation(getCursorContext(C), 
+    return cxloc::translateSourceLocation(getCursorContext(C), 
                                    getLocationFromExpr(getCursorExpr(C)));
 
-  if (!getCursorDecl(C)) {
-    CXSourceLocation empty = { 0, 0 };
-    return empty;
-  }
+  if (!getCursorDecl(C))
+    return clang_getNullLocation();
 
   Decl *D = getCursorDecl(C);
   SourceLocation Loc = D->getLocation();
   if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(D))
     Loc = Class->getClassLoc();
-  return translateSourceLocation(D->getASTContext(), Loc);
+  return cxloc::translateSourceLocation(D->getASTContext(), Loc);
 }
 
 CXSourceRange clang_getCursorExtent(CXCursor C) {
@@ -1492,25 +1562,25 @@ CXSourceRange clang_getCursorExtent(CXCursor C) {
       case CXCursor_ObjCSuperClassRef: {       
         std::pair<ObjCInterfaceDecl *, SourceLocation> P
           = getCursorObjCSuperClassRef(C);
-        return translateSourceRange(P.first->getASTContext(), P.second);
+        return cxloc::translateSourceRange(P.first->getASTContext(), P.second);
       }
         
       case CXCursor_ObjCProtocolRef: {       
         std::pair<ObjCProtocolDecl *, SourceLocation> P
           = getCursorObjCProtocolRef(C);
-        return translateSourceRange(P.first->getASTContext(), P.second);
+        return cxloc::translateSourceRange(P.first->getASTContext(), P.second);
       }
         
       case CXCursor_ObjCClassRef: {       
         std::pair<ObjCInterfaceDecl *, SourceLocation> P
           = getCursorObjCClassRef(C);
         
-        return translateSourceRange(P.first->getASTContext(), P.second);
+        return cxloc::translateSourceRange(P.first->getASTContext(), P.second);
       }
 
       case CXCursor_TypeRef: {       
         std::pair<TypeDecl *, SourceLocation> P = getCursorTypeRef(C);
-        return translateSourceRange(P.first->getASTContext(), P.second);
+        return cxloc::translateSourceRange(P.first->getASTContext(), P.second);
       }
         
       default:
@@ -1520,20 +1590,18 @@ CXSourceRange clang_getCursorExtent(CXCursor C) {
   }
 
   if (clang_isExpression(C.kind))
-    return translateSourceRange(getCursorContext(C), 
+    return cxloc::translateSourceRange(getCursorContext(C), 
                                 getCursorExpr(C)->getSourceRange());
 
   if (clang_isStatement(C.kind))
-    return translateSourceRange(getCursorContext(C), 
+    return cxloc::translateSourceRange(getCursorContext(C), 
                                 getCursorStmt(C)->getSourceRange());
   
-  if (!getCursorDecl(C)) {
-    CXSourceRange empty = { 0, 0, 0 };
-    return empty;
-  }
+  if (!getCursorDecl(C))
+    return clang_getNullRange();
   
   Decl *D = getCursorDecl(C);
-  return translateSourceRange(D->getASTContext(), D->getSourceRange());
+  return cxloc::translateSourceRange(D->getASTContext(), D->getSourceRange());
 }
 
 CXCursor clang_getCursorReferenced(CXCursor C) {
@@ -1643,7 +1711,7 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   case Decl::CXXRecord:
   case Decl::ClassTemplateSpecialization:
   case Decl::ClassTemplatePartialSpecialization:
-    if (TagDecl *Def = cast<TagDecl>(D)->getDefinition(D->getASTContext()))
+    if (TagDecl *Def = cast<TagDecl>(D)->getDefinition())
       return MakeCXCursor(Def, CXXUnit);
     return clang_getNullCursor();
 
@@ -1659,23 +1727,10 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   }
 
   case Decl::Var: {
-    VarDecl *Var = cast<VarDecl>(D);
-
-    // Variables with initializers have definitions.
-    const VarDecl *Def = 0;
-    if (Var->getDefinition(Def))
-      return MakeCXCursor(const_cast<VarDecl *>(Def), CXXUnit);
-
-    // extern and private_extern variables are not definitions.
-    if (Var->hasExternalStorage())
-      return clang_getNullCursor();
-
-    // In-line static data members do not have definitions.
-    if (Var->isStaticDataMember() && !Var->isOutOfLine())
-      return clang_getNullCursor();
-
-    // All other variables are themselves definitions.
-    return C;
+    // Ask the variable if it has a definition.
+    if (VarDecl *Def = cast<VarDecl>(D)->getDefinition())
+      return MakeCXCursor(Def, CXXUnit);
+    return clang_getNullCursor();
   }
    
   case Decl::FunctionTemplate: {
@@ -1687,7 +1742,7 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
    
   case Decl::ClassTemplate: {
     if (RecordDecl *Def = cast<ClassTemplateDecl>(D)->getTemplatedDecl()
-                                          ->getDefinition(D->getASTContext()))
+                                                            ->getDefinition())
       return MakeCXCursor(
                          cast<CXXRecordDecl>(Def)->getDescribedClassTemplate(), 
                           CXXUnit);
@@ -1843,6 +1898,247 @@ void clang_getDefinitionSpellingAndExtent(CXCursor C,
 } // end: extern "C"
 
 //===----------------------------------------------------------------------===//
+// Token-based Operations.
+//===----------------------------------------------------------------------===//
+
+/* CXToken layout:
+ *   int_data[0]: a CXTokenKind
+ *   int_data[1]: starting token location
+ *   int_data[2]: token length
+ *   int_data[3]: reserved
+ *   ptr_data: for identifiers and keywords, an IdentifierInfo*. 
+ *   otherwise unused.
+ */
+extern "C" {
+
+CXTokenKind clang_getTokenKind(CXToken CXTok) {
+  return static_cast<CXTokenKind>(CXTok.int_data[0]);
+}
+
+CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) {
+  switch (clang_getTokenKind(CXTok)) {
+  case CXToken_Identifier:
+  case CXToken_Keyword:
+    // We know we have an IdentifierInfo*, so use that.
+    return CIndexer::createCXString(
+              static_cast<IdentifierInfo *>(CXTok.ptr_data)->getNameStart());
+
+  case CXToken_Literal: {
+    // We have stashed the starting pointer in the ptr_data field. Use it.
+    const char *Text = static_cast<const char *>(CXTok.ptr_data);
+    return CIndexer::createCXString(llvm::StringRef(Text, CXTok.int_data[2]), 
+                                    true);
+  }
+      
+  case CXToken_Punctuation:
+  case CXToken_Comment:
+    break;
+  }
+  
+  // We have to find the starting buffer pointer the hard way, by 
+  // deconstructing the source location.
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  if (!CXXUnit)
+    return CIndexer::createCXString("");
+  
+  SourceLocation Loc = SourceLocation::getFromRawEncoding(CXTok.int_data[1]);
+  std::pair<FileID, unsigned> LocInfo
+    = CXXUnit->getSourceManager().getDecomposedLoc(Loc);
+  std::pair<const char *,const char *> Buffer
+    = CXXUnit->getSourceManager().getBufferData(LocInfo.first);
+
+  return CIndexer::createCXString(llvm::StringRef(Buffer.first+LocInfo.second,
+                                                  CXTok.int_data[2]), 
+                                  true);
+}
+ 
+CXSourceLocation clang_getTokenLocation(CXTranslationUnit TU, CXToken CXTok) {
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  if (!CXXUnit)
+    return clang_getNullLocation();
+  
+  return cxloc::translateSourceLocation(CXXUnit->getASTContext(),
+                        SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+}
+
+CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) {
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  if (!CXXUnit)
+    return clang_getNullRange();
+  
+  return cxloc::translateSourceRange(CXXUnit->getASTContext(), 
+                        SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+}
+  
+void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
+                    CXToken **Tokens, unsigned *NumTokens) {
+  if (Tokens)
+    *Tokens = 0;
+  if (NumTokens)
+    *NumTokens = 0;
+  
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  if (!CXXUnit || !Tokens || !NumTokens)
+    return;
+  
+  SourceRange R = cxloc::translateCXSourceRange(Range);
+  if (R.isInvalid())
+    return;
+  
+  SourceManager &SourceMgr = CXXUnit->getSourceManager();
+  std::pair<FileID, unsigned> BeginLocInfo
+    = SourceMgr.getDecomposedLoc(R.getBegin());
+  std::pair<FileID, unsigned> EndLocInfo
+    = SourceMgr.getDecomposedLoc(R.getEnd());
+  
+  // Cannot tokenize across files.
+  if (BeginLocInfo.first != EndLocInfo.first)
+    return;
+  
+  // Create a lexer 
+  std::pair<const char *,const char *> Buffer
+    = SourceMgr.getBufferData(BeginLocInfo.first);
+  Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
+            CXXUnit->getASTContext().getLangOptions(),
+            Buffer.first, Buffer.first + BeginLocInfo.second, Buffer.second);
+  Lex.SetCommentRetentionState(true);
+  
+  // Lex tokens until we hit the end of the range.
+  const char *EffectiveBufferEnd = Buffer.first + EndLocInfo.second;
+  llvm::SmallVector<CXToken, 32> CXTokens;
+  Token Tok;
+  do {
+    // Lex the next token
+    Lex.LexFromRawLexer(Tok);
+    if (Tok.is(tok::eof))
+      break;
+    
+    // Initialize the CXToken.
+    CXToken CXTok;
+    
+    //   - Common fields
+    CXTok.int_data[1] = Tok.getLocation().getRawEncoding();
+    CXTok.int_data[2] = Tok.getLength();
+    CXTok.int_data[3] = 0;
+    
+    //   - Kind-specific fields
+    if (Tok.isLiteral()) {
+      CXTok.int_data[0] = CXToken_Literal;
+      CXTok.ptr_data = (void *)Tok.getLiteralData();
+    } else if (Tok.is(tok::identifier)) {
+      // Lookup the identifier to determine whether we have a 
+      std::pair<FileID, unsigned> LocInfo
+        = SourceMgr.getDecomposedLoc(Tok.getLocation());
+      const char *StartPos 
+        = CXXUnit->getSourceManager().getBufferData(LocInfo.first).first + 
+          LocInfo.second;
+      IdentifierInfo *II
+        = CXXUnit->getPreprocessor().LookUpIdentifierInfo(Tok, StartPos);
+      CXTok.int_data[0] = II->getTokenID() == tok::identifier?
+                               CXToken_Identifier
+                             : CXToken_Keyword;
+      CXTok.ptr_data = II;
+    } else if (Tok.is(tok::comment)) {
+      CXTok.int_data[0] = CXToken_Comment;
+      CXTok.ptr_data = 0;
+    } else {
+      CXTok.int_data[0] = CXToken_Punctuation;
+      CXTok.ptr_data = 0;
+    }
+    CXTokens.push_back(CXTok);
+  } while (Lex.getBufferLocation() <= EffectiveBufferEnd);
+  
+  if (CXTokens.empty())
+    return;
+  
+  *Tokens = (CXToken *)malloc(sizeof(CXToken) * CXTokens.size());
+  memmove(*Tokens, CXTokens.data(), sizeof(CXToken) * CXTokens.size());
+  *NumTokens = CXTokens.size();
+}
+
+typedef llvm::DenseMap<unsigned, CXCursor> AnnotateTokensData;
+
+enum CXChildVisitResult AnnotateTokensVisitor(CXCursor cursor, 
+                                              CXCursor parent, 
+                                              CXClientData client_data) {
+  AnnotateTokensData *Data = static_cast<AnnotateTokensData *>(client_data);
+
+  // We only annotate the locations of declarations, simple
+  // references, and expressions which directly reference something.
+  CXCursorKind Kind = clang_getCursorKind(cursor);
+  if (clang_isDeclaration(Kind) || clang_isReference(Kind)) {
+    // Okay: We can annotate the location of this declaration with the
+    // declaration or reference
+  } else if (clang_isExpression(cursor.kind)) {
+    if (Kind != CXCursor_DeclRefExpr &&
+        Kind != CXCursor_MemberRefExpr &&
+        Kind != CXCursor_ObjCMessageExpr)
+      return CXChildVisit_Recurse;
+
+    CXCursor Referenced = clang_getCursorReferenced(cursor);
+    if (Referenced == cursor || Referenced == clang_getNullCursor())
+      return CXChildVisit_Recurse;
+    
+    // Okay: we can annotate the location of this expression
+  } else {
+    // Nothing to annotate
+    return CXChildVisit_Recurse;
+  }
+  
+  CXSourceLocation Loc = clang_getCursorLocation(cursor);
+  (*Data)[Loc.int_data] = cursor;
+  return CXChildVisit_Recurse;
+}
+
+void clang_annotateTokens(CXTranslationUnit TU,
+                          CXToken *Tokens, unsigned NumTokens,
+                          CXCursor *Cursors) {
+  if (NumTokens == 0)
+    return;
+
+  // Any token we don't specifically annotate will have a NULL cursor.
+  for (unsigned I = 0; I != NumTokens; ++I)
+    Cursors[I] = clang_getNullCursor();
+
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  if (!CXXUnit || !Tokens)
+    return;
+
+  // Annotate all of the source locations in the region of interest that map 
+  SourceRange RegionOfInterest;
+  RegionOfInterest.setBegin(
+        cxloc::translateSourceLocation(clang_getTokenLocation(TU, Tokens[0])));
+  SourceLocation End
+    = cxloc::translateSourceLocation(clang_getTokenLocation(TU, 
+                                                     Tokens[NumTokens - 1]));
+  RegionOfInterest.setEnd(CXXUnit->getPreprocessor().getLocForEndOfToken(End));
+  // FIXME: Would be great to have a "hint" cursor, then walk from that
+  // hint cursor upward until we find a cursor whose source range encloses
+  // the region of interest, rather than starting from the translation unit.
+  AnnotateTokensData Annotated;
+  CXCursor Parent = clang_getTranslationUnitCursor(CXXUnit);
+  CursorVisitor AnnotateVis(CXXUnit, AnnotateTokensVisitor, &Annotated, 
+                            Decl::MaxPCHLevel, RegionOfInterest);
+  AnnotateVis.VisitChildren(Parent);
+
+  for (unsigned I = 0; I != NumTokens; ++I) {
+    // Determine whether we saw a cursor at this token's location.
+    AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]);
+    if (Pos == Annotated.end())
+      continue;
+
+    Cursors[I] = Pos->second;
+  }
+}
+
+void clang_disposeTokens(CXTranslationUnit TU, 
+                         CXToken *Tokens, unsigned NumTokens) {
+  free(Tokens);
+}
+  
+} // end: extern "C"
+
+//===----------------------------------------------------------------------===//
 // CXString Operations.
 //===----------------------------------------------------------------------===//
 
@@ -1864,8 +2160,8 @@ void clang_disposeString(CXString string) {
   
 extern "C" {
 
-const char *clang_getClangVersion() {
-  return getClangFullVersion();
+CXString clang_getClangVersion() {
+  return CIndexer::createCXString(getClangFullVersion(), true);
 }
 
 } // end: extern "C"
diff --git a/tools/CIndex/CIndex.exports b/tools/CIndex/CIndex.exports
index d349086b9c11..2c738190b4ad 100644
--- a/tools/CIndex/CIndex.exports
+++ b/tools/CIndex/CIndex.exports
@@ -1,3 +1,4 @@
+_clang_annotateTokens
 _clang_codeComplete
 _clang_createIndex
 _clang_createTranslationUnit
@@ -5,6 +6,7 @@ _clang_createTranslationUnitFromSourceFile
 _clang_disposeCodeCompleteResults
 _clang_disposeIndex
 _clang_disposeString
+_clang_disposeTokens
 _clang_disposeTranslationUnit
 _clang_equalCursors
 _clang_equalLocations
@@ -23,17 +25,33 @@ _clang_getCursorReferenced
 _clang_getCursorSpelling
 _clang_getCursorUSR
 _clang_getDefinitionSpellingAndExtent
+_clang_getDiagnosticFixItInsertion
+_clang_getDiagnosticFixItKind
+_clang_getDiagnosticFixItRemoval
+_clang_getDiagnosticFixItReplacement
+_clang_getDiagnosticLocation
+_clang_getDiagnosticNumFixIts
+_clang_getDiagnosticNumRanges
+_clang_getDiagnosticRange
+_clang_getDiagnosticSeverity
+_clang_getDiagnosticSpelling
 _clang_getFile
 _clang_getFileName
 _clang_getFileTime
+_clang_getInclusions
 _clang_getInstantiationLocation
 _clang_getLocation
 _clang_getNullCursor
 _clang_getNullLocation
+_clang_getNullRange
 _clang_getNumCompletionChunks
 _clang_getRange
 _clang_getRangeEnd
 _clang_getRangeStart
+_clang_getTokenExtent
+_clang_getTokenKind
+_clang_getTokenLocation
+_clang_getTokenSpelling
 _clang_getTranslationUnitCursor
 _clang_getTranslationUnitSpelling
 _clang_isCursorDefinition
@@ -44,4 +62,5 @@ _clang_isReference
 _clang_isStatement
 _clang_isTranslationUnit
 _clang_setUseExternalASTGeneration
+_clang_tokenize
 _clang_visitChildren
diff --git a/tools/CIndex/CIndexCodeCompletion.cpp b/tools/CIndex/CIndexCodeCompletion.cpp
index f3b60dc8b01a..4e41c5f1c7b5 100644
--- a/tools/CIndex/CIndexCodeCompletion.cpp
+++ b/tools/CIndex/CIndexCodeCompletion.cpp
@@ -13,6 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "CIndexer.h"
+#include "CIndexDiagnostic.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -176,6 +178,8 @@ struct AllocatedCXCodeCompleteResults : public CXCodeCompleteResults {
   /// \brief The memory buffer from which we parsed the results. We
   /// retain this buffer because the completion strings point into it.
   llvm::MemoryBuffer *Buffer;
+
+  LangOptions LangOpts;
 };
 
 CXCodeCompleteResults *clang_codeComplete(CXIndex CIdx,
@@ -186,10 +190,19 @@ CXCodeCompleteResults *clang_codeComplete(CXIndex CIdx,
                                           struct CXUnsavedFile *unsaved_files,
                                           const char *complete_filename,
                                           unsigned complete_line,
-                                          unsigned complete_column) {
+                                          unsigned complete_column,
+                                          CXDiagnosticCallback diag_callback,
+                                          CXClientData diag_client_data) {
   // The indexer, which is mainly used to determine where diagnostics go.
   CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
 
+  // Configure the diagnostics.
+  DiagnosticOptions DiagOpts;
+  llvm::OwningPtr<Diagnostic> Diags;
+  Diags.reset(CompilerInstance::createDiagnostics(DiagOpts, 0, 0));
+  CIndexDiagnosticClient DiagClient(diag_callback, diag_client_data);
+  Diags->setClient(&DiagClient);
+  
   // The set of temporary files that we've built.
   std::vector<llvm::sys::Path> TemporaryFiles;
 
@@ -220,7 +233,8 @@ CXCodeCompleteResults *clang_codeComplete(CXIndex CIdx,
   argv.push_back("-no-code-completion-debug-printer");
   argv.push_back("-Xclang");
   argv.push_back("-code-completion-macros");
-  
+  argv.push_back("-fdiagnostics-binary");
+
   // Remap any unsaved files to temporary files.
   std::vector<std::string> RemapArgs;
   if (RemapFiles(num_unsaved_files, unsaved_files, RemapArgs, TemporaryFiles))
@@ -256,31 +270,39 @@ CXCodeCompleteResults *clang_codeComplete(CXIndex CIdx,
   // Add the null terminator.
   argv.push_back(NULL);
 
-  // Generate a temporary name for the AST file.
+  // Generate a temporary name for the code-completion results file.
   char tmpFile[L_tmpnam];
   char *tmpFileName = tmpnam(tmpFile);
   llvm::sys::Path ResultsFile(tmpFileName);
   TemporaryFiles.push_back(ResultsFile);
 
+  // Generate a temporary name for the diagnostics file.
+  char tmpFileResults[L_tmpnam];
+  char *tmpResultsFileName = tmpnam(tmpFileResults);
+  llvm::sys::Path DiagnosticsFile(tmpResultsFileName);
+  TemporaryFiles.push_back(DiagnosticsFile);
+
   // Invoke 'clang'.
   llvm::sys::Path DevNull; // leave empty, causes redirection to /dev/null
                            // on Unix or NUL (Windows).
   std::string ErrMsg;
-  const llvm::sys::Path *Redirects[] = { &DevNull, &ResultsFile, &DevNull, 0 };
+  const llvm::sys::Path *Redirects[] = { &DevNull, &ResultsFile, 
+                                         &DiagnosticsFile, 0 };
   llvm::sys::Program::ExecuteAndWait(ClangPath, &argv[0], /* env */ NULL,
                                      /* redirects */ &Redirects[0],
                                      /* secondsToWait */ 0,
                                      /* memoryLimits */ 0, &ErrMsg);
 
-  if (CXXIdx->getDisplayDiagnostics() && !ErrMsg.empty()) {
-    llvm::errs() << "clang_codeComplete: " << ErrMsg
-                 << '\n' << "Arguments: \n";
+  if (!ErrMsg.empty()) {
+    std::string AllArgs;
     for (std::vector<const char*>::iterator I = argv.begin(), E = argv.end();
-         I!=E; ++I) {
+         I != E; ++I) {
+      AllArgs += ' ';
       if (*I)
-        llvm::errs() << ' ' << *I << '\n';
+        AllArgs += *I;
     }
-    llvm::errs() << '\n';
+    
+    Diags->Report(diag::err_fe_clang) << AllArgs << ErrMsg;
   }
 
   // Parse the resulting source file to find code-completion results.
@@ -319,6 +341,13 @@ CXCodeCompleteResults *clang_codeComplete(CXIndex CIdx,
     Results->Buffer = F;
   }
 
+  // FIXME: The LangOptions we are passing here are not at all correct. However,
+  // in the current design we must pass something in so the SourceLocations have
+  // a LangOptions object to refer to.
+  ReportSerializedDiagnostics(DiagnosticsFile, *Diags, 
+                              num_unsaved_files, unsaved_files,
+                              Results->LangOpts);
+  
   for (unsigned i = 0, e = TemporaryFiles.size(); i != e; ++i)
     TemporaryFiles[i].eraseFromDisk();
 
diff --git a/tools/CIndex/CIndexDiagnostic.cpp b/tools/CIndex/CIndexDiagnostic.cpp
new file mode 100644
index 000000000000..d26094f7ee6f
--- /dev/null
+++ b/tools/CIndex/CIndexDiagnostic.cpp
@@ -0,0 +1,260 @@
+/*===-- CIndexDiagnostics.cpp - Diagnostics C Interface -----------*- C -*-===*\
+|*                                                                            *|
+|*                     The LLVM Compiler Infrastructure                       *|
+|*                                                                            *|
+|* This file is distributed under the University of Illinois Open Source      *|
+|* License. See LICENSE.TXT for details.                                      *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* Implements the diagnostic functions of the Clang C interface.              *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+#include "CIndexDiagnostic.h"
+#include "CIndexer.h"
+#include "CXSourceLocation.h"
+
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace clang;
+using namespace clang::cxloc;
+
+//-----------------------------------------------------------------------------
+// Opaque data structures                        
+//-----------------------------------------------------------------------------
+namespace {
+  /// \brief The storage behind a CXDiagnostic
+  struct CXStoredDiagnostic {
+    /// \brief The translation unit this diagnostic came from.
+    const LangOptions *LangOptsPtr;
+    
+    /// \brief The severity level of this diagnostic.
+    Diagnostic::Level Level;
+    
+    /// \brief A reference to the diagnostic information.
+    const DiagnosticInfo &Info;
+  };
+}
+
+//-----------------------------------------------------------------------------
+// CIndex Diagnostic Client                        
+//-----------------------------------------------------------------------------
+CIndexDiagnosticClient::~CIndexDiagnosticClient() { }
+
+void CIndexDiagnosticClient::BeginSourceFile(const LangOptions &LangOpts,
+                                             const Preprocessor *PP) {
+  assert(!LangOptsPtr && "Invalid state!");
+  LangOptsPtr = &LangOpts;
+}
+
+void CIndexDiagnosticClient::EndSourceFile() {
+  assert(LangOptsPtr && "Invalid state!");
+  LangOptsPtr = 0;
+}
+
+void CIndexDiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
+                                              const DiagnosticInfo &Info) {
+  if (!Callback)
+    return;
+
+  assert((LangOptsPtr || Info.getLocation().isInvalid()) &&
+         "Missing language options with located diagnostic!");
+  CXStoredDiagnostic Stored = { this->LangOptsPtr, DiagLevel, Info };
+  Callback(&Stored, ClientData);
+}
+
+//-----------------------------------------------------------------------------
+// C Interface Routines                        
+//-----------------------------------------------------------------------------
+extern "C" {
+  
+enum CXDiagnosticSeverity clang_getDiagnosticSeverity(CXDiagnostic Diag) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag)
+    return CXDiagnostic_Ignored;
+  
+  switch (StoredDiag->Level) {
+  case Diagnostic::Ignored: return CXDiagnostic_Ignored;
+  case Diagnostic::Note:    return CXDiagnostic_Note;
+  case Diagnostic::Warning: return CXDiagnostic_Warning;
+  case Diagnostic::Error:   return CXDiagnostic_Error;
+  case Diagnostic::Fatal:   return CXDiagnostic_Fatal;
+  }
+ 
+  llvm_unreachable("Invalid diagnostic level");
+  return CXDiagnostic_Ignored;
+}
+  
+CXSourceLocation clang_getDiagnosticLocation(CXDiagnostic Diag) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || StoredDiag->Info.getLocation().isInvalid())
+    return clang_getNullLocation();
+  
+  return translateSourceLocation(StoredDiag->Info.getLocation().getManager(),
+                                 *StoredDiag->LangOptsPtr,
+                                 StoredDiag->Info.getLocation());
+}
+
+CXString clang_getDiagnosticSpelling(CXDiagnostic Diag) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag)
+    return CIndexer::createCXString("");
+  
+  llvm::SmallString<64> Spelling;
+  StoredDiag->Info.FormatDiagnostic(Spelling);
+  return CIndexer::createCXString(Spelling.str(), true);
+}
+
+unsigned clang_getDiagnosticNumRanges(CXDiagnostic Diag) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || StoredDiag->Info.getLocation().isInvalid())
+    return 0;
+  
+  return StoredDiag->Info.getNumRanges();
+}
+  
+CXSourceRange clang_getDiagnosticRange(CXDiagnostic Diag, unsigned Range) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || Range >= StoredDiag->Info.getNumRanges() || 
+      StoredDiag->Info.getLocation().isInvalid())
+    return clang_getNullRange();
+  
+  return translateSourceRange(StoredDiag->Info.getLocation().getManager(),
+                              *StoredDiag->LangOptsPtr,
+                              StoredDiag->Info.getRange(Range));
+}
+
+unsigned clang_getDiagnosticNumFixIts(CXDiagnostic Diag) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag)
+    return 0;
+  
+  return StoredDiag->Info.getNumCodeModificationHints();
+}
+
+enum CXFixItKind clang_getDiagnosticFixItKind(CXDiagnostic Diag, 
+                                              unsigned FixIt) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || FixIt >= StoredDiag->Info.getNumCodeModificationHints())
+    return CXFixIt_Insertion;
+  
+  const CodeModificationHint &Hint
+    = StoredDiag->Info.getCodeModificationHint(FixIt);
+  if (Hint.RemoveRange.isInvalid())
+    return CXFixIt_Insertion;
+  if (Hint.InsertionLoc.isInvalid())
+    return CXFixIt_Removal;
+  
+  return CXFixIt_Replacement;  
+}
+
+CXString clang_getDiagnosticFixItInsertion(CXDiagnostic Diag, 
+                                           unsigned FixIt,
+                                           CXSourceLocation *Location) {
+  if (Location)
+    *Location = clang_getNullLocation();
+
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || FixIt >= StoredDiag->Info.getNumCodeModificationHints())
+    return CIndexer::createCXString("");
+  
+  const CodeModificationHint &Hint
+    = StoredDiag->Info.getCodeModificationHint(FixIt);
+
+  if (Location && StoredDiag->Info.getLocation().isValid())
+    *Location = translateSourceLocation(
+                                    StoredDiag->Info.getLocation().getManager(),
+                                        *StoredDiag->LangOptsPtr,
+                                        Hint.InsertionLoc);
+  return CIndexer::createCXString(Hint.CodeToInsert);
+}
+
+CXSourceRange clang_getDiagnosticFixItRemoval(CXDiagnostic Diag, 
+                                              unsigned FixIt) {
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || FixIt >= StoredDiag->Info.getNumCodeModificationHints() ||
+      StoredDiag->Info.getLocation().isInvalid())
+    return clang_getNullRange();
+  
+  const CodeModificationHint &Hint
+    = StoredDiag->Info.getCodeModificationHint(FixIt);
+  return translateSourceRange(StoredDiag->Info.getLocation().getManager(),
+                              *StoredDiag->LangOptsPtr,
+                              Hint.RemoveRange);
+}
+
+CXString clang_getDiagnosticFixItReplacement(CXDiagnostic Diag, 
+                                             unsigned FixIt,
+                                             CXSourceRange *Range) {
+  if (Range)
+    *Range = clang_getNullRange();
+
+  CXStoredDiagnostic *StoredDiag = static_cast<CXStoredDiagnostic *>(Diag);
+  if (!StoredDiag || FixIt >= StoredDiag->Info.getNumCodeModificationHints() ||
+      StoredDiag->Info.getLocation().isInvalid()) {
+    if (Range)
+      *Range = clang_getNullRange();
+    
+    return CIndexer::createCXString("");
+  }
+  
+  const CodeModificationHint &Hint
+    = StoredDiag->Info.getCodeModificationHint(FixIt);
+  if (Range)
+    *Range = translateSourceRange(StoredDiag->Info.getLocation().getManager(),
+                                  *StoredDiag->LangOptsPtr,
+                                  Hint.RemoveRange);
+  return CIndexer::createCXString(Hint.CodeToInsert);  
+}
+  
+} // end extern "C"
+
+void clang::ReportSerializedDiagnostics(const llvm::sys::Path &DiagnosticsPath,
+                                        Diagnostic &Diags,
+                                        unsigned num_unsaved_files,
+                                        struct CXUnsavedFile *unsaved_files,
+                                        const LangOptions &LangOpts) {
+  using llvm::MemoryBuffer;
+  using llvm::StringRef;
+  MemoryBuffer *F = MemoryBuffer::getFile(DiagnosticsPath.c_str());
+  if (!F)
+    return;
+
+  // Enter the unsaved files into the file manager.
+  SourceManager SourceMgr;
+  FileManager FileMgr;
+  for (unsigned I = 0; I != num_unsaved_files; ++I) {
+    const FileEntry *File = FileMgr.getVirtualFile(unsaved_files[I].Filename,
+                                                   unsaved_files[I].Length,
+                                                   0);
+    if (!File) {
+      Diags.Report(diag::err_fe_remap_missing_from_file)
+        << unsaved_files[I].Filename;
+      return;
+    }
+
+    MemoryBuffer *Buffer
+      = MemoryBuffer::getMemBuffer(unsaved_files[I].Contents,
+                           unsaved_files[I].Contents + unsaved_files[I].Length);
+    if (!Buffer)
+      return;
+
+    SourceMgr.overrideFileContents(File, Buffer);
+  }
+
+  Diags.getClient()->BeginSourceFile(LangOpts, 0);
+
+  // Parse the diagnostics, emitting them one by one until we've
+  // exhausted the data.
+  StringRef Buffer = F->getBuffer();
+  const char *Memory = Buffer.data(), *MemoryEnd = Memory + Buffer.size();
+  while (Memory != MemoryEnd) {
+    DiagnosticBuilder DB = Diags.Deserialize(FileMgr, SourceMgr, 
+                                             Memory, MemoryEnd);
+    if (!DB.isActive())
+      return;
+  }
+
+  Diags.getClient()->EndSourceFile();
+}
diff --git a/tools/CIndex/CIndexDiagnostic.h b/tools/CIndex/CIndexDiagnostic.h
new file mode 100644
index 000000000000..9f7ae51a1083
--- /dev/null
+++ b/tools/CIndex/CIndexDiagnostic.h
@@ -0,0 +1,66 @@
+/*===-- CIndexDiagnostic.h - Diagnostics C Interface ------------*- C++ -*-===*\
+|*                                                                            *|
+|*                     The LLVM Compiler Infrastructure                       *|
+|*                                                                            *|
+|* This file is distributed under the University of Illinois Open Source      *|
+|* License. See LICENSE.TXT for details.                                      *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* Implements the diagnostic functions of the Clang C interface.              *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+#ifndef LLVM_CLANG_CINDEX_DIAGNOSTIC_H
+#define LLVM_CLANG_CINDEX_DIAGNOSTIC_H
+
+#include "clang-c/Index.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/LangOptions.h"
+
+namespace llvm { namespace sys {
+class Path;
+} }
+
+namespace clang {
+
+class Diagnostic;
+class LangOptions;
+class Preprocessor;
+  
+/**
+ * \brief Diagnostic client that translates Clang diagnostics into diagnostics
+ * for the C interface to Clang.
+ */
+class CIndexDiagnosticClient : public DiagnosticClient {
+  CXDiagnosticCallback Callback;
+  CXClientData ClientData;
+  const LangOptions *LangOptsPtr;
+  
+public:
+  CIndexDiagnosticClient(CXDiagnosticCallback Callback,
+                         CXClientData ClientData)
+    : Callback(Callback), ClientData(ClientData), LangOptsPtr(0) { }
+  
+  virtual ~CIndexDiagnosticClient();
+  
+  virtual void BeginSourceFile(const LangOptions &LangOpts,
+                               const Preprocessor *PP);
+  
+  virtual void EndSourceFile();
+
+  virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
+                                const DiagnosticInfo &Info);
+};
+
+/// \brief Given the path to a file that contains binary, serialized
+/// diagnostics produced by Clang, emit those diagnostics via the
+/// given diagnostic engine.
+void ReportSerializedDiagnostics(const llvm::sys::Path &DiagnosticsPath,
+                                 Diagnostic &Diags,
+                                 unsigned num_unsaved_files,
+                                 struct CXUnsavedFile *unsaved_files,
+                                 const LangOptions &LangOpts);
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_CINDEX_DIAGNOSTIC_H
diff --git a/tools/CIndex/CIndexInclusionStack.cpp b/tools/CIndex/CIndexInclusionStack.cpp
new file mode 100644
index 000000000000..4a884e588432
--- /dev/null
+++ b/tools/CIndex/CIndexInclusionStack.cpp
@@ -0,0 +1,65 @@
+//===- CIndexInclusionStack.cpp - Clang-C Source Indexing Library ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a callback mechanism for clients to get the inclusion
+// stack from a translation unit.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIndexer.h"
+#include "CXSourceLocation.h"
+#include "clang/AST/DeclVisitor.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
+
+extern "C" {
+void clang_getInclusions(CXTranslationUnit TU, CXInclusionVisitor CB,
+                         CXClientData clientData) {
+  
+  ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+  SourceManager &SM = CXXUnit->getSourceManager();
+  ASTContext &Ctx = CXXUnit->getASTContext();
+
+  llvm::SmallVector<CXSourceLocation, 10> InclusionStack;
+  unsigned i = SM.sloc_loaded_entry_size();
+  unsigned n =  SM.sloc_entry_size();
+
+  // In the case where all the SLocEntries are in an external source, traverse
+  // those SLocEntries as well.  This is the case where we are looking
+  // at the inclusion stack of an AST/PCH file.
+  if (i >= n)
+    i = 0;
+  
+  for ( ; i < n ; ++i) {
+
+    const SrcMgr::SLocEntry &SL = SM.getSLocEntry(i);
+    
+    if (!SL.isFile())
+      continue;
+
+    const SrcMgr::FileInfo &FI = SL.getFile();
+    if (!FI.getContentCache()->Entry)
+      continue;
+    
+    // Build the inclusion stack.
+    SourceLocation L = FI.getIncludeLoc();
+    InclusionStack.clear();
+    while (L.isValid()) {
+      PresumedLoc PLoc = SM.getPresumedLoc(L);
+      InclusionStack.push_back(cxloc::translateSourceLocation(Ctx, L));
+      L = PLoc.getIncludeLoc();
+    }
+            
+    // Callback to the client.
+    // FIXME: We should have a function to construct CXFiles.
+    CB((CXFile) FI.getContentCache()->Entry, 
+       InclusionStack.data(), InclusionStack.size(), clientData);
+  }    
+}
+} // end extern C
diff --git a/tools/CIndex/CIndexUSRs.cpp b/tools/CIndex/CIndexUSRs.cpp
index fd605fb68d1b..a992dbf12a00 100644
--- a/tools/CIndex/CIndexUSRs.cpp
+++ b/tools/CIndex/CIndexUSRs.cpp
@@ -74,7 +74,7 @@ void USRGenerator::VisitFunctionDecl(FunctionDecl *D) {
 void USRGenerator::VisitNamedDecl(NamedDecl *D) {
   VisitDeclContext(D->getDeclContext());
   const std::string &s = D->getNameAsString();
-  assert(!s.empty());
+//  assert(!s.empty());
   Out << "@^" << s;
 }
 
diff --git a/tools/CIndex/CIndexer.cpp b/tools/CIndex/CIndexer.cpp
index 53636a4ff351..0774ae2f3722 100644
--- a/tools/CIndex/CIndexer.cpp
+++ b/tools/CIndex/CIndexer.cpp
@@ -69,7 +69,7 @@ const llvm::sys::Path& CIndexer::getClangPath() {
 
   // We now have the CIndex directory, locate clang relative to it.
   CIndexPath.eraseComponent();
-  CIndexPath.eraseComponent();
+  CIndexPath.appendComponent("..");
   CIndexPath.appendComponent("bin");
   CIndexPath.appendComponent("clang");
 #endif
@@ -95,16 +95,38 @@ std::string CIndexer::getClangResourcesPath() {
   return P.str();
 }
 
+static llvm::sys::Path GetTemporaryPath() {
+  // FIXME: This is lame; sys::Path should provide this function (in particular,
+  // it should know how to find the temporary files dir).
+  std::string Error;
+  const char *TmpDir = ::getenv("TMPDIR");
+  if (!TmpDir)
+    TmpDir = ::getenv("TEMP");
+  if (!TmpDir)
+    TmpDir = ::getenv("TMP");
+  if (!TmpDir)
+    TmpDir = "/tmp";
+  llvm::sys::Path P(TmpDir);
+  P.appendComponent("remap");
+  if (P.makeUnique(false, &Error))
+    return llvm::sys::Path("");
+
+  // FIXME: Grumble, makeUnique sometimes leaves the file around!?  PR3837.
+  P.eraseFromDisk(false, 0);
+
+  return P;
+}
+
 bool clang::RemapFiles(unsigned num_unsaved_files,
                        struct CXUnsavedFile *unsaved_files,
                        std::vector<std::string> &RemapArgs,
                        std::vector<llvm::sys::Path> &TemporaryFiles) {
   for (unsigned i = 0; i != num_unsaved_files; ++i) {
-    char tmpFile[L_tmpnam];
-    char *tmpFileName = tmpnam(tmpFile);
-    
     // Write the contents of this unsaved file into the temporary file.
-    llvm::sys::Path SavedFile(tmpFileName);
+    llvm::sys::Path SavedFile(GetTemporaryPath());
+    if (SavedFile.empty())
+      return true;
+
     std::string ErrorInfo;
     llvm::raw_fd_ostream OS(SavedFile.c_str(), ErrorInfo);
     if (!ErrorInfo.empty())
@@ -120,7 +142,7 @@ bool clang::RemapFiles(unsigned num_unsaved_files,
     // Remap the file.
     std::string RemapArg = unsaved_files[i].Filename;
     RemapArg += ';';
-    RemapArg += tmpFileName;
+    RemapArg += SavedFile.str();
     RemapArgs.push_back("-Xclang");
     RemapArgs.push_back("-remap-file");
     RemapArgs.push_back("-Xclang");
diff --git a/tools/CIndex/CIndexer.h b/tools/CIndex/CIndexer.h
index d01454f9dc64..b83e2b7f326c 100644
--- a/tools/CIndex/CIndexer.h
+++ b/tools/CIndex/CIndexer.h
@@ -18,36 +18,20 @@
 #include "clang-c/Index.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/ASTUnit.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/System/Path.h"
 #include <vector>
 
 using namespace clang;
 
-/// IgnoreDiagnosticsClient - A DiagnosticsClient that just ignores emitted
-/// warnings and errors.
-class IgnoreDiagnosticsClient : public DiagnosticClient {
-public:
-  virtual ~IgnoreDiagnosticsClient() {}
-  virtual void HandleDiagnostic(Diagnostic::Level, const DiagnosticInfo &) {}
-};
-
 class CIndexer {
-  DiagnosticOptions DiagOpts;
-  IgnoreDiagnosticsClient IgnoreDiagClient;
-  llvm::OwningPtr<Diagnostic> TextDiags;
-  Diagnostic IgnoreDiags;
   bool UseExternalASTGeneration;
   bool OnlyLocalDecls;
-  bool DisplayDiagnostics;
   
   llvm::sys::Path ClangPath;
   
 public:
-  CIndexer() : IgnoreDiags(&IgnoreDiagClient), UseExternalASTGeneration(false),
-               OnlyLocalDecls(false), DisplayDiagnostics(false) 
-  {
-    TextDiags.reset(CompilerInstance::createDiagnostics(DiagOpts, 0, 0));
-  }
+  CIndexer() : UseExternalASTGeneration(false), OnlyLocalDecls(false) { }
   
   /// \brief Whether we only want to see "local" declarations (that did not
   /// come from a previous precompiled header). If false, we want to see all
@@ -55,20 +39,11 @@ public:
   bool getOnlyLocalDecls() const { return OnlyLocalDecls; }
   void setOnlyLocalDecls(bool Local = true) { OnlyLocalDecls = Local; }
   
-  bool getDisplayDiagnostics() const { return DisplayDiagnostics; }
-  void setDisplayDiagnostics(bool Display = true) {
-    DisplayDiagnostics = Display;
-  }
-  
   bool getUseExternalASTGeneration() const { return UseExternalASTGeneration; }
   void setUseExternalASTGeneration(bool Value) {
     UseExternalASTGeneration = Value;
   }
   
-  Diagnostic &getDiags() {
-    return DisplayDiagnostics ? *TextDiags : IgnoreDiags;
-  }
-  
   /// \brief Get the path of the clang binary.
   const llvm::sys::Path& getClangPath();
   
@@ -76,6 +51,8 @@ public:
   std::string getClangResourcesPath();
 
   static CXString createCXString(const char *String, bool DupString = false);
+  static CXString createCXString(llvm::StringRef String, 
+                                 bool DupString = false);
 };
 
 namespace clang {
diff --git a/tools/CIndex/CMakeLists.txt b/tools/CIndex/CMakeLists.txt
index 730eaafb5a2e..26e1b3bb905c 100644
--- a/tools/CIndex/CMakeLists.txt
+++ b/tools/CIndex/CMakeLists.txt
@@ -21,6 +21,8 @@ set( LLVM_LINK_COMPONENTS
 add_clang_library(CIndex
   CIndex.cpp
   CIndexCodeCompletion.cpp
+  CIndexDiagnostic.cpp
+  CIndexInclusionStack.cpp
   CIndexUSRs.cpp
   CIndexer.cpp
   CXCursor.cpp
diff --git a/tools/CIndex/CXCursor.cpp b/tools/CIndex/CXCursor.cpp
index 63d9bc9f2e9a..ec1477eb8a27 100644
--- a/tools/CIndex/CXCursor.cpp
+++ b/tools/CIndex/CXCursor.cpp
@@ -29,6 +29,7 @@ CXCursor cxcursor::MakeCXCursorInvalid(CXCursorKind K) {
 }
 
 static CXCursorKind GetCursorKind(Decl *D) {
+  assert(D && "Invalid arguments!");
   switch (D->getKind()) {
     case Decl::Enum:               return CXCursor_EnumDecl;
     case Decl::EnumConstant:       return CXCursor_EnumConstantDecl;
@@ -72,11 +73,13 @@ static CXCursorKind GetCursorKind(Decl *D) {
 }
 
 CXCursor cxcursor::MakeCXCursor(Decl *D, ASTUnit *TU) {
+  assert(D && TU && "Invalid arguments!");
   CXCursor C = { GetCursorKind(D), { D, 0, TU } };
   return C;
 }
 
 CXCursor cxcursor::MakeCXCursor(Stmt *S, Decl *Parent, ASTUnit *TU) {
+  assert(S && TU && "Invalid arguments!");
   CXCursorKind K = CXCursor_NotImplemented;
   
   switch (S->getStmtClass()) {
@@ -112,7 +115,6 @@ CXCursor cxcursor::MakeCXCursor(Stmt *S, Decl *Parent, ASTUnit *TU) {
     K = CXCursor_UnexposedStmt;
     break;
       
-  case Stmt::ExprClass:
   case Stmt::PredefinedExprClass:        
   case Stmt::IntegerLiteralClass:        
   case Stmt::FloatingLiteralClass:       
@@ -123,12 +125,10 @@ CXCursor cxcursor::MakeCXCursor(Stmt *S, Decl *Parent, ASTUnit *TU) {
   case Stmt::UnaryOperatorClass:         
   case Stmt::SizeOfAlignOfExprClass:     
   case Stmt::ArraySubscriptExprClass:    
-  case Stmt::CastExprClass:              
   case Stmt::BinaryOperatorClass:        
   case Stmt::CompoundAssignOperatorClass:
   case Stmt::ConditionalOperatorClass:   
   case Stmt::ImplicitCastExprClass:
-  case Stmt::ExplicitCastExprClass:
   case Stmt::CStyleCastExprClass:
   case Stmt::CompoundLiteralExprClass:   
   case Stmt::ExtVectorElementExprClass:  
@@ -162,6 +162,7 @@ CXCursor cxcursor::MakeCXCursor(Stmt *S, Decl *Parent, ASTUnit *TU) {
   case Stmt::UnaryTypeTraitExprClass:     
   case Stmt::DependentScopeDeclRefExprClass:  
   case Stmt::CXXBindTemporaryExprClass:   
+  case Stmt::CXXBindReferenceExprClass:   
   case Stmt::CXXExprWithTemporariesClass: 
   case Stmt::CXXUnresolvedConstructExprClass:
   case Stmt::CXXDependentScopeMemberExprClass:
@@ -214,6 +215,7 @@ CXCursor cxcursor::MakeCXCursor(Stmt *S, Decl *Parent, ASTUnit *TU) {
 CXCursor cxcursor::MakeCursorObjCSuperClassRef(ObjCInterfaceDecl *Super, 
                                                SourceLocation Loc, 
                                                ASTUnit *TU) {
+  assert(Super && TU && "Invalid arguments!");
   void *RawLoc = reinterpret_cast<void *>(Loc.getRawEncoding());
   CXCursor C = { CXCursor_ObjCSuperClassRef, { Super, RawLoc, TU } };
   return C;    
@@ -230,6 +232,7 @@ cxcursor::getCursorObjCSuperClassRef(CXCursor C) {
 CXCursor cxcursor::MakeCursorObjCProtocolRef(ObjCProtocolDecl *Super, 
                                              SourceLocation Loc, 
                                              ASTUnit *TU) {
+  assert(Super && TU && "Invalid arguments!");
   void *RawLoc = reinterpret_cast<void *>(Loc.getRawEncoding());
   CXCursor C = { CXCursor_ObjCProtocolRef, { Super, RawLoc, TU } };
   return C;    
@@ -246,6 +249,7 @@ cxcursor::getCursorObjCProtocolRef(CXCursor C) {
 CXCursor cxcursor::MakeCursorObjCClassRef(ObjCInterfaceDecl *Class, 
                                           SourceLocation Loc, 
                                           ASTUnit *TU) {
+  assert(Class && TU && "Invalid arguments!");
   void *RawLoc = reinterpret_cast<void *>(Loc.getRawEncoding());
   CXCursor C = { CXCursor_ObjCClassRef, { Class, RawLoc, TU } };
   return C;    
@@ -261,6 +265,7 @@ cxcursor::getCursorObjCClassRef(CXCursor C) {
 
 CXCursor cxcursor::MakeCursorTypeRef(TypeDecl *Type, SourceLocation Loc, 
                                      ASTUnit *TU) {
+  assert(Type && TU && "Invalid arguments!");
   void *RawLoc = reinterpret_cast<void *>(Loc.getRawEncoding());
   CXCursor C = { CXCursor_TypeRef, { Type, RawLoc, TU } };
   return C;    
diff --git a/tools/CIndex/CXCursor.h b/tools/CIndex/CXCursor.h
index 546dd7f48543..30fb26c936bf 100644
--- a/tools/CIndex/CXCursor.h
+++ b/tools/CIndex/CXCursor.h
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_CXCURSOR_H
-#define LLVM_CLANG_CXCursor_H
+#define LLVM_CLANG_CXCURSOR_H
 
 #include "clang-c/Index.h"
 #include "clang/Basic/SourceLocation.h"
diff --git a/tools/CIndex/CXSourceLocation.h b/tools/CIndex/CXSourceLocation.h
new file mode 100644
index 000000000000..8bfc6f9da7ab
--- /dev/null
+++ b/tools/CIndex/CXSourceLocation.h
@@ -0,0 +1,76 @@
+//===- CXSourceLocation.h - CXSourceLocations Utilities ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines routines for manipulating CXSourceLocations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_CXSOURCELOCATION_H
+#define LLVM_CLANG_CXSOURCELOCATION_H
+
+#include "clang-c/Index.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/AST/ASTContext.h"
+
+namespace clang {
+      
+class ASTContext;
+
+namespace cxloc {
+
+/// \brief Translate a Clang source location into a CIndex source location.
+static inline CXSourceLocation 
+translateSourceLocation(const SourceManager &SM, const LangOptions &LangOpts,
+                        SourceLocation Loc) {
+  CXSourceLocation Result = { { (void*) &SM, (void*) &LangOpts, },
+                              Loc.getRawEncoding() };
+  return Result;
+}
+  
+/// \brief Translate a Clang source location into a CIndex source location.
+static inline CXSourceLocation translateSourceLocation(ASTContext &Context,
+                                                       SourceLocation Loc) {
+  return translateSourceLocation(Context.getSourceManager(),
+                                 Context.getLangOptions(),
+                                 Loc);
+}
+
+/// \brief Translate a Clang source range into a CIndex source range.
+///
+/// Clang internally represents ranges where the end location points to the
+/// start of the token at the end. However, for external clients it is more
+/// useful to have a CXSourceRange be a proper half-open interval. This routine
+/// does the appropriate translation.
+CXSourceRange translateSourceRange(const SourceManager &SM, 
+                                   const LangOptions &LangOpts,
+                                   SourceRange R);
+  
+/// \brief Translate a Clang source range into a CIndex source range.
+static inline CXSourceRange translateSourceRange(ASTContext &Context,
+                                                 SourceRange R) {
+  return translateSourceRange(Context.getSourceManager(),
+                              Context.getLangOptions(),
+                              R);
+}
+
+static inline SourceLocation translateSourceLocation(CXSourceLocation L) {
+  return SourceLocation::getFromRawEncoding(L.int_data);
+}
+
+static inline SourceRange translateCXSourceRange(CXSourceRange R) {
+  return SourceRange(SourceLocation::getFromRawEncoding(R.begin_int_data),
+                     SourceLocation::getFromRawEncoding(R.end_int_data));
+}
+
+
+}} // end namespace: clang::cxloc
+
+#endif
diff --git a/tools/CIndex/Makefile b/tools/CIndex/Makefile
index 7bdbba172f2c..3e288623e5eb 100644
--- a/tools/CIndex/Makefile
+++ b/tools/CIndex/Makefile
@@ -11,7 +11,6 @@ LEVEL = ../../../..
 LIBRARYNAME = CIndex
 
 CPPFLAGS += -I$(PROJ_SRC_DIR)/../../include -I$(PROJ_OBJ_DIR)/../../include
-CXXFLAGS = -fno-rtti
 
 # Include this here so we can get the configuration of the targets
 # that have been configured for construction. We have to do this