Merge pull request #306 from SiegeLord/more_rust_updates

data/filetypes.rust

@@ -22,7 +22,7 @@ lexerror=error
 
 [keywords]
 # all items must be in one line
-primary=alignof as be box break const continue crate do else enum extern false fn for if impl in let loop match mod mut offsetof once priv proc pub pure ref return self sizeof static struct super trait true type typeof unsafe unsized use while yield
+primary=alignof as be box break const continue crate do else enum extern false fn for if impl in let loop match mod mut offsetof once priv proc pub pure ref return self sizeof static struct super trait true type typeof unsafe unsized use virtual while yield
 secondary=bool char f32 f64 i16 i32 i64 i8 int str u16 u32 u64 u8 uint
 tertiary=Self
 

tagmanager/ctags/rust.c

@@ -24,7 +24,7 @@
 /*
 *   MACROS
 */
-#define MAX_STRING_LENGTH 64
+#define MAX_STRING_LENGTH 256
 
 /*
 *   DATA DECLARATIONS
@@ -117,9 +117,7 @@ static void writeCurTokenToStr (lexerState *lexer, vString *out_str)
 			vStringCat(out_str, lexer->token_str);
 			break;
 		case TOKEN_STRING:
-			vStringPut(out_str, '"');
 			vStringCat(out_str, lexer->token_str);
-			vStringPut(out_str, '"');
 			break;
 		case TOKEN_WHITESPACE:
 			vStringPut(out_str, ' ');
@@ -152,6 +150,14 @@ static void advanceNChar (lexerState *lexer, int n)
 		advanceChar(lexer);
 }
 
+/* Store the current character in lexerState::token_str if there is space
+ * (set by MAX_STRING_LENGTH), and then read the next character from the file */
+static void advanceAndStoreChar (lexerState *lexer)
+{
+	if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
+		vStringPut(lexer->token_str, (char) lexer->cur_c);
+	advanceChar(lexer);
+}
 
 static boolean isWhitespace (int c)
 {
@@ -182,19 +188,30 @@ static void scanWhitespace (lexerState *lexer)
 }
 
 /* Normal line comments start with two /'s and continue until the next \n
- * (NOT any other newline character!). Additionally, a shebang in the beginning
- * of the file also counts as a line comment.
+ * (potentially after a \r). Additionally, a shebang in the beginning of the
+ * file also counts as a line comment as long as it is not this sequence: #![ .
  * Block comments start with / followed by a * and end with a * followed by a /.
  * Unlike in C/C++ they nest. */
 static void scanComments (lexerState *lexer)
 {
-	/* // or #! */
-	if (lexer->next_c == '/' || lexer->next_c == '!')
+	/* // */
+	if (lexer->next_c == '/')
 	{
 		advanceNChar(lexer, 2);
 		while (lexer->cur_c != EOF && lexer->cur_c != '\n')
 			advanceChar(lexer);
 	}
+	/* #! */
+	else if (lexer->next_c == '!')
+	{
+		advanceNChar(lexer, 2);
+		/* If it is exactly #![ then it is not a comment, but an attribute */
+		if (lexer->cur_c == '[')
+			return;
+		while (lexer->cur_c != EOF && lexer->cur_c != '\n')
+			advanceChar(lexer);
+	}
+	/* block comment */
 	else if (lexer->next_c == '*')
 	{
 		int level = 1;
@@ -224,8 +241,7 @@ static void scanIdentifier (lexerState *lexer)
 	vStringClear(lexer->token_str);
 	do
 	{
-		vStringPut(lexer->token_str, (char) lexer->cur_c);
-		advanceChar(lexer);
+		advanceAndStoreChar(lexer);
 	} while(lexer->cur_c != EOF && isIdentifierContinue(lexer->cur_c));
 }
 
@@ -237,16 +253,14 @@ static void scanIdentifier (lexerState *lexer)
 static void scanString (lexerState *lexer)
 {
 	vStringClear(lexer->token_str);
-	advanceChar(lexer);
+	advanceAndStoreChar(lexer);
 	while (lexer->cur_c != EOF && lexer->cur_c != '"')
 	{
 		if (lexer->cur_c == '\\' && lexer->next_c == '"')
-			advanceChar(lexer);
-		if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
-			vStringPut(lexer->token_str, (char) lexer->cur_c);
-		advanceChar(lexer);
+			advanceAndStoreChar(lexer);
+		advanceAndStoreChar(lexer);
 	}
-	advanceChar(lexer);
+	advanceAndStoreChar(lexer);
 }
 
 /* Raw strings look like this: r"" or r##""## where the number of
@@ -255,52 +269,75 @@ static void scanRawString (lexerState *lexer)
 {
 	size_t num_initial_hashes = 0;
 	vStringClear(lexer->token_str);
-	advanceChar(lexer);
+	advanceAndStoreChar(lexer);
 	/* Count how many leading hashes there are */
 	while (lexer->cur_c == '#')
 	{
 		num_initial_hashes++;
-		advanceChar(lexer);
+		advanceAndStoreChar(lexer);
 	}
 	if (lexer->cur_c != '"')
 		return;
-	advanceChar(lexer);
+	advanceAndStoreChar(lexer);
 	while (lexer->cur_c != EOF)
 	{
-		if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
-			vStringPut(lexer->token_str, (char) lexer->cur_c);
 		/* Count how many trailing hashes there are. If the number is equal or more
 		 * than the number of leading hashes, break. */
 		if (lexer->cur_c == '"')
 		{
 			size_t num_trailing_hashes = 0;
-			advanceChar(lexer);
+			advanceAndStoreChar(lexer);
 			while (lexer->cur_c == '#' && num_trailing_hashes < num_initial_hashes)
 			{
 				num_trailing_hashes++;
 
-				if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
-					vStringPut(lexer->token_str, (char) lexer->cur_c);
-				advanceChar(lexer);
+				advanceAndStoreChar(lexer);
 			}
 			if (num_trailing_hashes == num_initial_hashes)
-			{
-				/* Strip the trailing hashes and quotes */
-				if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH && vStringLength(lexer->token_str) > num_trailing_hashes + 1)
-				{
-					lexer->token_str->length = vStringLength(lexer->token_str) - num_trailing_hashes - 1;
-					lexer->token_str->buffer[lexer->token_str->length] = '\0';
-				}
 				break;
-			}
 		}
 		else
 		{
-			advanceChar(lexer);
+			advanceAndStoreChar(lexer);
 		}
 	}
 }
 
+/* This deals with character literals: 'n', '\n', '\uFFFF'; and lifetimes:
+ * 'lifetime. We'll use this approximate regexp for the literals:
+ * \' \\ [^']+ \' or \' [^'] \' or \' \\ \' \'. Either way, we'll treat this
+ * token as a string, so it gets preserved as is for function signatures with
+ * lifetimes. */
+static void scanCharacterOrLifetime (lexerState *lexer)
+{
+	vStringClear(lexer->token_str);
+	advanceAndStoreChar(lexer);
+
+	if (lexer->cur_c == '\\')
+	{
+		advanceAndStoreChar(lexer);
+		/* The \' \\ \' \' (literally '\'') case */
+		if (lexer->cur_c == '\'' && lexer->next_c == '\'')
+		{
+			advanceAndStoreChar(lexer);
+			advanceAndStoreChar(lexer);
+		}
+		/* The \' \\ [^']+ \' case */
+		else
+		{
+			while (lexer->cur_c != EOF && lexer->cur_c != '\'')
+				advanceAndStoreChar(lexer);
+		}
+	}
+	/* The \' [^'] \' case */
+	else if (lexer->cur_c != '\'' && lexer->next_c == '\'')
+	{
+		advanceAndStoreChar(lexer);
+		advanceAndStoreChar(lexer);
+	}
+	/* Otherwise it is malformed, or a lifetime */
+}
+
 /* Advances the parser one token, optionally skipping whitespace
  * (otherwise it is concatenated and returned as a single whitespace token).
  * Whitespace is needed to properly render function signatures. Unrecognized
@@ -343,6 +380,11 @@ static int advanceToken (lexerState *lexer, boolean skip_whitspace)
 			scanRawString(lexer);
 			return lexer->cur_token = TOKEN_STRING;
 		}
+		else if (lexer->cur_c == '\'')
+		{
+			scanCharacterOrLifetime(lexer);
+			return lexer->cur_token = TOKEN_STRING;
+		}
 		else if (isIdentifierStart(lexer->cur_c))
 		{
 			scanIdentifier(lexer);

tests/ctags/test_input.rs

 #! fn ignored_in_comment() {}
-#[feature(globs)];
-#[feature(macro_rules)];
+#![feature(globs)]
+#![feature(macro_rules)]
 use std::*;
 use std::io::stdio::println;
 use test_input2::*;
 mod test_input2;
 
+fn lifetime_and_char<'lifetime>(_: &'lifetime int)
+{
+	let s = '"';
+	let s = '}';
+	let s = '\'';
+	let s = '\uffff';
+	fn not_hidden_by_char() {}
+}
+
+fn preserve_string_delims(_bar: extern r#"C"# fn()) {}
+
 pub struct A
 {
 	foo: fn() -> int,
@@ -44,18 +55,18 @@ macro_rules! test_macro
 
 macro_rules! ignore (($($x:tt)*) => (()))
 
-fn yada(a:int,c:Foo,b:test_input2::fruit::SomeStruct) -> ~str {
-	a.to_str()
+fn yada(a:int,c:Foo,b:test_input2::fruit::SomeStruct) -> String {
+	a.to_string()
 }
 
 fn main() {	
 	use test_input2::fruit::*;	
-	io::println(foo_bar_test_func(SomeStruct{red_value:1,green_value:2,blue_value:3},(4,5)).to_str());
+	io::println(foo_bar_test_func(SomeStruct{red_value:1,green_value:2,blue_value:3},(4,5)).to_string().as_slice());
 	let a=Foo{foo_field_1:2};
 	a.my_method(1);
 	let c=a_cat(3);
 	let d=Foo{foo_field_1:a.foo_field_1+2}; a.test();
-	println(a.foo_field_1.to_str());
+	println(a.foo_field_1.to_string().as_slice());
 	ignore!
 	(
 		fn ignored_inside_macro() {}
@@ -111,21 +122,21 @@ trait DoZ {
 
 impl Testable for Foo {
 	fn test(&self) {
-		println(self.foo_field_1.to_str());
+		println(self.foo_field_1.to_string().as_slice());
 	}
 
 	fn test1(&self) {
-		println(self.foo_field_1.to_str());
+		println(self.foo_field_1.to_string().as_slice());
 	}
 
 	fn test2(&self) {
-		println(self.foo_field_1.to_str());
+		println(self.foo_field_1.to_string().as_slice());
 	}
 }
 
 impl DoZ for Foo {
 	fn do_z(&self) {
-		println(self.foo_field_1.to_str());
+		println(self.foo_field_1.to_string().as_slice());
 	}
 }
 

tests/ctags/test_input.rs.tags

@@ -27,10 +27,13 @@ foo
 foo_field_18Foo0
 gfunc16<X:Testable+DoZ>(x:&X)0
 ignore655360
+lifetime_and_char16<'lifetime>(_: &'lifetime int)0
 main16()0
 my_method128(&self,_:int)Foo0
 nested16()main0
+not_hidden_by_char16()lifetime_and_char0
 only_field8S10
+preserve_string_delims16(_bar: extern r#"C"# fn())0
 size163840
 some216(a:Animal)0
 test128(&self)Foo0
@@ -46,4 +49,4 @@ test_macro
 x8Foo20
 x8TraitedStructTest0
 y8Foo20
-yada16(a:int,c:Foo,b:test_input2::fruit::SomeStruct) -> ~str0
+yada16(a:int,c:Foo,b:test_input2::fruit::SomeStruct) -> String0

tests/ctags/test_input2.rs

-use std::io::stdio::println;
+#![cfg(not(test))] fn not_hashbang() {}
 
 pub fn foo_bar_test_func(apples:fruit::SomeStruct,(oranges,lemon):(int,int))->int{
+	use std::io::stdio::println;
 	let some_var_name=2*oranges;
 	let a=SomeLongStructName{v:0};
 	println("a");println("b");	println("c");

tests/ctags/test_input2.rs.tags

@@ -14,6 +14,7 @@ granite
 green_value8fruit::SomeStruct0
 limestone16()mineral0
 mineral2560
+not_hashbang16()0
 red_value8fruit::SomeStruct0
 v8SomeLongStructName0
 veg2560