Writing Records
After you have obtained the IRepositoryRecordInserter or IRepositoryRecordInserter2 interface (as described in Getting and Putting Data), you need to generate the data structures that you are going to write. Before you begin writing, make sure you understand the record data structures (see Repository Record Data Structures) and are able to construct them efficiently. The repository API provides two ways to write records: you can use either complete record structures or arrays of the smaller structures from which records are made up. The next steps depend on this choice.
Approach 1: Writing Whole Record Structures
In this approach, you combine your newly generated tags and contents structures into complete record structures, put the records in an array and supply the array to the PutRecords method of the IRepositoryRecordInserter interface.
Approach 2: Splitting Records for Writing
This approach requires that you plan in advance which of your record fields best to use as tags. This will enable you to create records with shared tags and different contents. The IRepositoryRecordInserterLight interface stores the tags that you want to share among your records. To get this interface, call the BindFields method of the IRepositoryRecordInserter interface you have obtained. This method accepts your tags as a parameter.
After that, supply the contents parts of your records to the IRepositoryRecordInserterLight interface; it will form complete record structures and perform the writing.
Example (C#)
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Runtime.InteropServices;
using REPOSITORYSERVICESLib;
using REPOSITORYRECORDINSERTERLib;
using INTRUSTENVIRONMENTLib;
namespace RepositoryRecordInserterTest2
{
class Program
{
public static uint ToUnixTime(DateTime date)
{
var epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
return (uint)Convert.ToInt64((date.ToUniversalTime() - epoch).TotalSeconds);
}
public static DateTime UnixTimestampToDateTime(uint unixTime)
{
DateTime unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc);
long unixTimeStampInTicks = (long)(unixTime * TimeSpan.TicksPerSecond);
return new DateTime(unixStart.Ticks + unixTimeStampInTicks);
}
class MyObserver : IDisposable, REPOSITORYSERVICESLib.IObserver
{
private AutoResetEvent m_waitHandler;
public int event_count;
public REPOSITORYSERVICESLib.ICookie m_cookie;
public MyObserver(AutoResetEvent x)
{
m_waitHandler = x;
event_count = 0;
}
public void OnDone()
{
Console.WriteLine("Search done");
m_waitHandler.Set();
}
public void OnError(int hr, string description)
{
Console.WriteLine("Search error - {0}", description);
m_waitHandler.Set();
}
public void OnNext(object data)
{
if (data != null)
{
IBulkRecord bulk_event = (data as IBulkRecord);
List<record> records = bulk_event.GetRecords().Cast<record>().ToList<record>();
foreach (record my_record in records)
{
Console.WriteLine("next record");
Console.WriteLine(" time - {0}", UnixTimestampToDateTime(my_record.record_contents.gmt_time));
foreach (named_string my_named_string in my_record.record_contents.named_fields)
{
Console.WriteLine(" key - {0}, value - {1}", my_named_string.name, my_named_string.value);
}
++event_count;
}
}
System.Runtime.InteropServices.Marshal.FinalReleaseComObject(data);
}
public void Dispose()
{
m_cookie.Stop();
}
}
static tags construct_sharding_fields(string log)
{
// For details about using sharding keys, see the "Repository Record Data Structures" topic
tags tg = new tags();
tg.directory_tag_1 = "ShardingLevel1";
tg.directory_tag_2 = "ShardingLevel2";
tg.directory_tag_3 = "{A9E5C7A2-5C01-41B7-9D36-E562DFDDEFA9}"; // Sharding level 3 must be a GUID
tg.directory_tag_4 = log;
tg.file_tag_1 = 0;
tg.file_tag_2 = 500;
tg.file_tag_3 = 0;
tg.file_tag_4 = 0;
return tg;
}
static contents construct_contents_fields(insertion_string[] insertion_strings, named_string[] named_fields, string formatting_record_field)
{
contents ct = new contents();
ct.string_field_1 = "string_field_1_value";
ct.string_field_2 = "string_field_2_value";
ct.string_field_3 = "string_field_3_value";
ct.string_field_4 = "string_field_4_value";
ct.string_field_5 = "string_field_5_value";
ct.formatting_record_field = formatting_record_field;
ct.gmt_time = ToUnixTime(DateTime.Now);
ct.field_1 = 300;
ct.field_2 = 50;
ct.field_3 = 2;
ct.field_4 = 3;
ct.strings = insertion_strings;
ct.named_fields = named_fields;
return ct;
}
static record construct_record(uint index, string logname, insertion_string[] insertion_strings, named_string[] named_fields, string description)
{
return new record() {
record_path = construct_sharding_fields(logname),
record_contents = construct_contents_fields(insertion_strings, named_fields, description)
};
}
static void insert_records(IRepositoryRecordInserter pInserter)
{
DateTime start = DateTime.Now;
List<record> records = new List<record>();
for (uint i = 0; i != 16000; ++i)
{
insertion_string[] insertion_strings =
{
new insertion_string() { index = 1, value = "My" },
new insertion_string() { index = 2, value = "String value 2" },
new insertion_string() { index = 6, value = "Event" },
new insertion_string() { index = 7, value = "String value 7" }
};
named_string[] named_fields =
{
new named_string() { name = "FieldName1", value = "FieldValue1"},
new named_string() { name = "FieldName2", value = "FieldValue2"},
new named_string() { name = "FieldName3", value = "FieldValue3"},
new named_string() { name = "FieldName4", value = "FieldValue4"},
new named_string() { name = "FieldName5", value = "FieldValue5"},
};
records.Add(construct_record(i, "Log1", insertion_strings, named_fields, "This %1 %6 description"));
}
pInserter.PutRecords(records.ToArray());
pInserter.Commit();
}
static void insert_records_on_server(IRepositoryRecordInserter2 pInserter)
{
DateTime start = DateTime.Now;
List<record> records = new List<record>();
for (uint i = 0; i != 16000; ++i)
{
insertion_string[] insertion_strings =
{
new insertion_string() { index = 1, value = "My" },
new insertion_string() { index = 2, value = "String value 2" },
new insertion_string() { index = 6, value = "Event" },
new insertion_string() { index = 7, value = "String value 7" }
};
named_string[] named_fields =
{
new named_string() { name = "FieldName1", value = "FieldValue1"},
new named_string() { name = "FieldName2", value = "FieldValue2"},
new named_string() { name = "FieldName3", value = "FieldValue3"},
new named_string() { name = "FieldName4", value = "FieldValue4"},
new named_string() { name = "FieldName5", value = "FieldValue5"},
};
records.Add(construct_record(i, "Log1", insertion_strings, named_fields, "This %1 %6 description"));
}
pInserter.PutRecords(records.ToArray());
pInserter.Commit2(ToServerRepositoryCommitType);
}
static tags construct_naive_sharding_fields(string log)
{
// For details about using sharding keys, see the "Repository Record Data Structures" topic
tags tg = new tags();
tg.directory_tag_1 = "ShardingLevel1";
tg.directory_tag_2 = "ShardingLevel2";
tg.directory_tag_3 = "{A9E5C7A2-5C01-41B7-9D36-E562DFDDEFA9}"; // Sharding level 3 must be a GUID
tg.directory_tag_4 = log; // ShardingLevel4
return tg;
}
static contents construct_naive_contents_fields(named_string[] named_fields)
{
contents ct = new contents();
ct.gmt_time = ToUnixTime(DateTime.Now);
ct.named_fields = named_fields;
return ct;
}
static record construct_naive_record(uint index, string logname, named_string[] named_fields)
{
return new record()
{
record_path = construct_naive_sharding_fields(logname),
record_contents = construct_naive_contents_fields(named_fields)
};
}
static void insert_naive_records(IRepositoryRecordInserter pInserter)
{
DateTime start = DateTime.Now;
List<record> records = new List<record>();
for (uint i = 0; i != 16000; ++i)
{
named_string[] named_fields =
{
new named_string() { name = "FieldName1", value = "FieldValue1"},
new named_string() { name = "FieldName2", value = "FieldValue2"},
new named_string() { name = "FieldName3", value = "FieldValue3"},
new named_string() { name = "FieldName4", value = "FieldValue4"},
new named_string() { name = "FieldName5", value = "FieldValue5"},
};
records.Add(construct_naive_record(i, "Log1", named_fields));
}
pInserter.PutRecords(records.ToArray());
pInserter.Commit();
}
static void search_records(IInTrustRepository intrust_repository, string query)
{
IObservable observable = intrust_repository.Searcher.Search(query);
AutoResetEvent waitHandler = new AutoResetEvent(false);
MyObserver observer = new MyObserver(waitHandler);
observable.Subscribe(observer, out observer.m_cookie);
waitHandler.WaitOne();
}
static void records_example(IInTrustRepository intrust_repository)
{
IRepositoryRecordInserter pInserter = intrust_repository.Inserter;
// Insert records with strictly key-value data directly to the repository
insert_naive_records(pInserter);
search_records(intrust_repository, "(in( __AnyField, \"rei\", \"(\\\\b|\\\\W|^)FieldValue5\" ));");
// Insert records directly to the repository
insert_records(pInserter);
search_records(intrust_repository, "(in( __AnyField, \"rei\", \"(\\\\b|\\\\W|^)String value 7\" ));"
// Insert records by queuing them on the server
insert_records_on_server(pInserter as IRepositoryRecordInserter2);
System.Threading.ThreadSleep(60000);// We need to wait, because there will be a delay up to a minute before the event arrives in the repository
search_records(intrust_repository, "(in( __AnyField, \"rei\", \"(\\\\b|\\\\W|^)String value 7\" ));"
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Invalid argument count.\n");
Console.WriteLine("\tRepositoryRecordInserterTest.exe <InTrust Server Binding String> <Repository Name>\n");
return;
}
try
{
InTrustEnvironmentintrust_environment = new InTrustEnvironment();
IInTrustOrganization3 intrust_organization = intrust_environment.ConnectToServer(args[0]).Organization as IInTrustOrganization3;
IInTrustRepository3 intrust_repository = intrust_organization.Repositories2.Cast<IInTrustRepository3>().Where(x => x.Name == args[1]).First();
records_example(intrust_repository);
}
catch (Exception e)
{
Console.WriteLine("Error : {0}", e.ToString());
}
}
}
}
Writing Events
After you have obtained the IRepositoryRecordInserter interface (as described in Getting and Putting Data), take the following steps:
- Generate valid event_with_extensions structure instances; this structure is described in Event Record Data Structures.
- Through combined use of the IEventToRecordFormatter and IRepositoryRecordInserter interfaces, supply your newly generated events for writing.
For that, use the Format method of the IEventToRecordFormatter interface to wrap your events in IBulkEventWithReadExtensions, and pass the resulting wrapper interface as a parameter to the PutRecords2 method of the IRepositoryRecordInserter interface.
Sort Order for Writing
When you write events in batches, the events must be sorted by gmt_time, but not necessarily throughout the entire batch. The important thing is to sort those events where the following are the same: domain, computer, log name. That is the scope where you need to sort. For example, if your event batch contains 1000 events from 1000 computers, no sorting is necessary. But if it is 1000 events from two computers, you need to do two sorts.
Out-of-Order Writing
Submitting events out of order is possible, but there is a serious caveat. Whenever the timestamp of your event is less than that of the event you submitted last, you must use the Commit method of your inserter interface before you write the “flashback” event. Here are some implications of this:
- Events that are newer than the last-submitted event and older than that event should not be put in the same batch. Otherwise, timestamps will not be interpreted correctly. For example, searching within a specific time range will not return events that match but were written out of order between commits.
- A huge amount of repository files (caused by frequent commits) is bad for performance. Batch your “old” events as much as possible.
Repository Record Data Structures
A record is a chunk of data that is (or has been prepared for being) stored in a repository and processed by repository searches. A record is made up of two parts: tags and contents. The tags indicate where in the file system the file with this record is located. At the same time, the tags double as record field values. The contents do not do anything other than contain record field values.
When you create records, it is important which fields you use as tags and which you use as contents. By handling tags rationally you can help speed up searches on the data you are storing and minimize disk usage by your repository contents.
When you search for records, it doesn't matter in search queries whether a value is used as a tag or as contents.
For recommendations on efficiently organizing data fields in records, see Recommendations on Setting Tags below.
tags
struct tags
{
BSTR directory_tag_1;
BSTR directory_tag_2;
BSTR directory_tag_3; // must specify a GUID
BSTR directory_tag_4;
unsigned file_tag_1;
unsigned file_tag_2;
unsigned file_tag_3;
unsigned file_tag_4;
};
Contains the values of eight of the record's fields. In addition to carrying record data, this combination of fields is used for identifying the path to a folder and the name of a file in the repository tree. Repository trees are four levels deep, and files are located only at the deepest level.
The directory tags specify the four nesting levels. The third level must be a string representation of a GUID; InTrust verifies the format. This requirement is due to the implementation of the repository services. The GUID is used for identifying event-providing data sources, and each data source type has a particular known GUID. You may want to come up with your own set of special-purpose GUIDs for your specific tasks (for example, hierarchical organization).
The file tags specify the four parts of the file's base name. A file represented by this structure contains one or more entries represented by contents and contents2 structures. If the number of entries per file hits a limit, the same base name is used for creating a new file, and the entries are continued in it.
In a simplified way, the location of a repository file in the file system hierarchy can be represented as follows:
repository_root
└ directory_tag_1
└ directory_tag_2
└ directory_tag_3
└ directory_tag_4
└ file_tag_1_file_tag_2_ file_tag_3_file_tag_4_InTrust_internal_tags
Because tags correlate with the file system, make sure their values do not contain characters that are disallowed in file and directory names.
contents
struct contents
{
unsigned field_1;
unsigned field_2;
short field_3;
short field_4;
DATE gmt_time;
BSTR string_field_1;
BSTR string_field_2;
BSTR string_field_3;
BSTR string_field_4;
BSTR string_field_5;
SAFEARRAY(struct insertion_string) strings;
SAFEARRAY(struct named_string) named_fields;
BSTR formatting_record_field;
};
Used for writing records to the reposiotry and contains the remaining values of the record fields in addition to those specified by the tags structure. This structure is physically represented by an entry in a repository file.
|
|
Note: The InTrust repository is known to easily handle up to 300 strings per record. Higher numbers of strings have not been tested and cannot be recommended. As a best practice, make sure that your string mapping is consistent; that is, the same string numbers should have the same meanings across your records. This is beneficial for repository searches. |
|
|
Caution: At this time, the field_2 field is not indexed and cannot be processed in repository searches. Writing useful data to this field is currently not recommended. |
contents2
struct contents2
{
unsigned field_1;
unsigned field_2;
short field_3;
short field_4;
DATE gmt_time;
BSTR string_field_1;
BSTR string_field_2;
BSTR string_field_3;
BSTR string_field_4;
BSTR string_field_5;
SAFEARRAY(struct insertion_string) strings;
SAFEARRAY(struct named_string) native_named_fields;
SAFEARRAY(struct named_string) resolved_named_fields;
BSTR formatting_record_field;
};
Used in results of repository searches and contains the remaining values of the record fields in addition to those specified by the tags structure. This structure is physically represented by an entry in a repository file.
Unlike the contents structure, the contents2 structure contains two distinct arrays of named_string structures. This is because the data for the native_named_fields field is not known during record writing. It is only available to repository searches.
|
|
Note: The InTrust repository is known to easily handle up to 300 strings per record. Higher numbers of strings have not been tested and cannot be recommended. As a best practice, make sure that your string mapping is consistent; that is, the same string numbers should have the same meanings across your records. This is beneficial for repository searches. |
|
|
Caution: At this time, the field_2 field is not indexed and cannot be processed in repository searches. Writing useful data to this field is currently not recommended. |
record
struct record
{
struct tags record_path;
struct contents record_contents;
};
Combines the path-specifying (tags) and complementary (contents) parts of a record into a single structure. This type of record is used for writing to the repository.
record2
struct record2
{
struct tags record_path;
struct contents2 record_contents;
};
Combines the path-specifying (tags) and complementary (contents2) parts of a record into a single structure. This type of record is returned by repository searches.
Recommendations on Setting Tags
Organize your record tags according to the likelihood of the value being the same in a given collection of records. That is, directory_tag_1 should contain the value that is the same in most of the records you are about to generate. Conversely, directory_tag_4 should contain the value that the fewest records have in common. Also note that the best way to map the tags (and thereby define how the records will be stored physically) is to make them correspond to something that falls into a meaningful hierarchy. For example, all your records might be Security log events, but it still doesn't make sense to make the log name the topmost level. You would do better to tag map directory_tag_1,..., directory_tag_4 to domain, computer, data source and log name, respectively; even though there is more value variation at the top levels this way.
A repository can store heterogeneous objects, but you need a way to tell their types apart. This requires a generic ID field, and directory_tag_3 is good for the purpose. If you come up with a GUID for each object type (file system, computer, Active Directory object and so on), you will not confuse them. A further improvement is to design a hierarchy of IDs.
Event Record Data Structures
These data structures are alternatives to generic repository record data structures. Use event records for convenience when your records represent log events. For details about the meaning of the fields used in event records, see Event Record Data Structures below.
The primary data structure is base_event. There are also two structures that extend it: event_with_extensions and event_with_read_extensions. For details about the use of these data structures, see Getting Events and Writing Events.
base_event
struct base_event
{
BSTR environment;
BSTR gathering_domain;
BSTR gathering_computer;
BSTR datasource_type;
BSTR gathered_event_log;
BSTR user_name;
BSTR user_domain;
BSTR source_name;
BSTR computer_name;
BSTR string_category;
BSTR description_template;
SAFEARRAY(struct insertion_string) strings;
SAFEARRAY(unsigned char) binary_data;
unsigned time_gmt;
unsigned time_generated;
long time_bias;
unsigned record_key;
unsigned event_id;
unsigned computer_type;
unsigned platform_id;
short version_major;
short version_minor;
short event_type;
short numeric_category;
unsigned padding000;
};
|
|
Caution: The binary_data field is present only for compatibility with Windows events. This data cannot be indexed or processed in repository searches. Writing useful data to this field is not recommended. |
event_with_read_extensions
struct event_with_read_extensions
{
struct base_event original_event;
BSTR formatted_description;
SAFEARRAY(struct augmented_insertion_string) resolved_strings;
SAFEARRAY(struct named_string) named_strings;
};
named_string
struct named_string
{
BSTR name;
BSTR value;
};
|
IMPORTANT: In the string names that you define, use only alphanumeric ASCII characters and the underscore (_) character. |
insertion_string
struct insertion_string
{
BSTR value;
int index;
int padding;
};
A regular insertion string.
augmented_insertion_string
struct augmented_insertion_string
{
int source_index;
int result_index;
BSTR value;
};
These are normalized parameters that are not originally present in native events. For a description of these parameters, see the Filter Parameters in Repository Viewer topic.
|
|
Note: The source_index field holds the index of the original insertion string. The result_index field holds the index of the resulting insertion string after the original has been resolved. |
event_with_extensions
struct event_with_extensions
{
struct base_event original_event;
SAFEARRAY(struct resolved_string) resolved_strings;
};
|
|
Note: The InTrust repository is known to easily handle up to 300 insertion strings per event. Higher numbers of strings have not been tested and cannot be recommended. As a best practice, make sure that your insertion string mapping is consistent; that is, the same insertion string numbers should have the same meanings across your events. This is beneficial for repository searches. |
event_with_extensions2
struct event_with_extensions2
{
struct base_event original_event;
SAFEARRAY(struct resolved_string) resolved_strings;
SAFEARRAY(struct named_string) named_fields;
};
resolved_string
struct resolved_string
{
BSTR value;
int insertion_string_index;
resolve_type insertion_string_resolve_type;
};
typedef enum
{
custom = 0,
parameter,
ad_object_guid_to_distinguished_name,
user_sid_to_user_name,
group_policy_guid_to_group_policy_object_name,
device_name_to_path
} resolve_type;
The resolve_type enumeration specifies what kind of resolution is supposed to have taken place to get the resulting value. The insertion string resolution mechanism in InTrust is fairly complex, but for the purposes of the repository service API it is enough to follow this example:
insertion_string[] insertion_strings =
{
new insertion_string() { index = 1, padding = 0, value = "original string" },
new insertion_string() { index = 2, padding = 0, value = "%%2308" }, // event parameter
new insertion_string() { index = 3, padding = 0, value = "{9F29FD37-3CD4-4179-99F1-A6341DCC4EB3}" }, // ad object guid (user guid)
new insertion_string() { index = 4, padding = 0, value = "S-1-1-0" }, // user sid
new insertion_string() { index = 5, padding = 0, value = "{29EDB5C5-B2C1-4001-9C96-EE51A6A7CAC3}" }, // ad object guid (group policy guid)
new insertion_string() { index = 6, padding = 0, value = "\\Device\\HarddiskVolume1\\SomeFolder\\SomeFile.txt" } };
resolved_string[] resolved_insertion_strings =
{
new resolved_string() {insertion_string_index = 2, insertion_string_resolve_type = resolve_type.parameter, value = "The user has not been granted the requested logon type at this machine."},
new resolved_string() {insertion_string_index = 3, insertion_string_resolve_type = resolve_type.ad_object_guid_to_distinguished_name, value = "CN=user1,DC=aa,DC=com"},
new resolved_string() {insertion_string_index = 4, insertion_string_resolve_type = resolve_type.user_sid_to_user_name, value = "EDM\\User2"},
new resolved_string() {insertion_string_index = 5, insertion_string_resolve_type = resolve_type.ad_object_guid_to_distinguished_name, value = "CN={B96B9D14-E2A4-47ae-8ACA-CB0460089616},DC=aa,DC=com"},
new resolved_string() {insertion_string_index = 5, insertion_string_resolve_type = resolve_type.group_policy_guid_to_group_policy_object_name, value = "MyGroupPolicyObject"},
new resolved_string() {insertion_string_index = 6, insertion_string_resolve_type = resolve_type.ad_object_guid_to_distinguished_name, value = "D:\\SomeFolder\\SomeFile.txt"},
};
Note that in the case of resolving a group policy GUID to a group policy name you need to provide two resolved_string instances.