Advanced Internet Search or AIS
is an program that speeds up the search process
this program goes to Google and looks want you gave it to search
then collect a number of results (number may differ when using different settings), each result will be analyzed by the application, if it found anything useful then it will be displayed in Results (see screenshot)
AIS comes with an build in Console to see what is has done or is doing (can be turned off), contents of the console can be copied or saved.
Screenshot:
Usage:
Main Form:
- In Internet Search objects you insert items (right click) that will be used to search with the search Engine
- In Page Search objects you insert items (right click) that will be used to search in pages
- Save Results
- Process: Start or Stop
- Settings
- Results: Result Count, Find unnecessary Items and remove them
Settings:
- Show Console
- Number of Urls that will be collect for each Internet Search Object (200 is around the first 20 pages of google results)(multiply by 10)
Console:
- Save Content (right click)
- Copy Content (right click)
Features:
- the whole process is now faster (not ± 20 min, but more 5 - 10 min for 200 Urls per Internet search object) then the older version
- better searching
- Save Results
- Settings included
- Results Handling
- Console display
Future Features:
- Chose which Search Engines will be used
AIS comes with an extra feature:
Filter list
this is a list of Items (separated by #) that will by replaced with nothing, in the Internet/Pages Search Objects
mostly useless words like: The
this list is still sort so any suggestions? (you can edit it yourself with notepad)
Need:
- Framework .net 2.0
this Program comes with a Curse though:
it reacts really bad to the stop command.
and it takes time to actually press the stop button
but if you have Patience you will reach it so don't freak out
Download
here is the code(VB)
THE WHOLE CREDIT GOES TO THE ORIGINAL PROGRAMMER --- Huntondoom
is an program that speeds up the search process
this program goes to Google and looks want you gave it to search
then collect a number of results (number may differ when using different settings), each result will be analyzed by the application, if it found anything useful then it will be displayed in Results (see screenshot)
AIS comes with an build in Console to see what is has done or is doing (can be turned off), contents of the console can be copied or saved.
Screenshot:
Main Form:
- In Internet Search objects you insert items (right click) that will be used to search with the search Engine
- In Page Search objects you insert items (right click) that will be used to search in pages
- Save Results
- Process: Start or Stop
- Settings
- Results: Result Count, Find unnecessary Items and remove them
Settings:
- Show Console
- Number of Urls that will be collect for each Internet Search Object (200 is around the first 20 pages of google results)(multiply by 10)
Console:
- Save Content (right click)
- Copy Content (right click)
Features:
- the whole process is now faster (not ± 20 min, but more 5 - 10 min for 200 Urls per Internet search object) then the older version
- better searching
- Save Results
- Settings included
- Results Handling
- Console display
Future Features:
- Chose which Search Engines will be used
AIS comes with an extra feature:
Filter list
this is a list of Items (separated by #) that will by replaced with nothing, in the Internet/Pages Search Objects
mostly useless words like: The
this list is still sort so any suggestions? (you can edit it yourself with notepad)
Need:
- Framework .net 2.0
this Program comes with a Curse though:
it reacts really bad to the stop command.
and it takes time to actually press the stop button
but if you have Patience you will reach it so don't freak out
Download
here is the code(VB)
Imports System.IO
Imports System.Text
Public Class MainDialog
Public FoUr, Results, FilterOutList As New List(Of String)
Public SearchEngineFilterlist, ListofSearchEngine As New List(Of String)
Public UrlCount As Integer = 200
Public ProcesStop As Boolean = False
Public ShowConsole As Boolean = True
Public SearchEngine As String = "Google"
Public SearchEngineURL As String = ""
'Setup
Private Sub MainDialog_Load(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles MyBase.Load
ListofSearchEngine = GetListOfSearchEngine()
If ShowConsole = True Then Console.Show()
ProText.Text = ""
FilterOutList = GetFilterList()
SearchEngineURL = GetSearchEngineURL(SearchEngine)
SearchEngineFilterlist = GetSearchEngineFilterlist(SearchEngine)
UpdateStuff()
End Sub
'Proces
Private Sub StopToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles StopToolStripMenuItem.Click
ProcesStop = True
Application.DoEvents()
StartToolStripMenuItem.Enabled = True
ProText.Text = "Trying to kill Process"
ResetStuff()
Do Until ProcesStop = True
ProcesStop = True
Loop
End Sub
Private Sub StartToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles StartToolStripMenuItem.Click
'Voorbereiding
CheckSearchObjects()
If ShowConsole = True Then Console.Show()
If InternetSearchObjects.Items.Count = 0 Then Exit Sub
StartToolStripMenuItem.Enabled = False
SetProgressBar(InternetSearchObjects.Items.Count - 1, 0, 0)
MenuStrip1.Update()
UpdateStuff()
'Process I
For ItemCount As Integer = 0 To InternetSearchObjects.Items.Count - 1
Dim Total As Integer = 0
Dim Item As String = InternetSearchObjects.Items(ItemCount).ToString
For PageNumber As Integer = 0 To Math.Round((UrlCount / 10), 0)
UpdateStuff(ItemCount, "Getting Urls for: " & Item, "A.I.S.:" & Item)
Dim Page As String = GetSearchEnginePage(Item, PageNumber)
Dim Urls() As String = GetUrls(Page)
Urls = FilterUrls(Urls)
Total += UBound(Urls)
If ProcesStop = True Then
ResetStuff()
Exit Sub
End If
For Index As Integer = 0 To UBound(Urls)
If Urls(Index) <> "" Then FoUr.Add(Urls(Index))
If ShowConsole = True And Urls(Index) <> "" Then Console.Writeline(Urls(Index))
Next
If Total >= UrlCount Then Exit For
Next
Next
'Process II
SetProgressBar(FoUr.Count, 0, 0)
For UrlIndex As Integer = 0 To FoUr.Count - 1
If ProcesStop = True Then
ProcesStop = False
ResetStuff()
Exit Sub
End If
Dim Page As String = GetPage(FoUr(UrlIndex)).ToLower
Page = Page.ToLower
If ShowConsole = True Then Console.Writeline("Checking: " & FoUr(UrlIndex) & vbNewLine)
UpdateStuff(UrlIndex, "Checking: " & FoUr(UrlIndex), "A.I.S.:" & FoUr(UrlIndex))
Application.DoEvents()
Dim TmpRes As New List(Of String)
For I As Integer = 0 To PageSearchObject.Items.Count - 1
If Page.Contains(PageSearchObject.Items(I).ToString.ToLower) Then FoundInformation.Items.Add(PageSearchObject.Items(I))
TmpRes = GetString(PageSearchObject.Items(I).ToString.ToLower, Page)
For J As Integer = 0 To TmpRes.Count - 1
If DetectUrl(TmpRes(J), PageSearchObject.Items(I)) = False Then
FoundInformation.Items.Add(TmpRes(J) & "@ " & FoUr(UrlIndex))
Results.Add(TmpRes(J) & "@ " & FoUr(UrlIndex))
If ShowConsole = True Then Console.Writeline(TmpRes(J))
End If
If ProcesStop = True Then
ResetStuff()
ProcesStop = False
Exit Sub
End If
Next
Next
Next
ResetStuff()
End Sub
'Handelingen
' InSeOb
Private Sub InsertToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles InsertToolStripMenuItem.Click
Dim Add As String = InputBox("Search Object to be used in Search Engine" & vbCrLf & "Seperate Items are Created when you use: ,", "Adding a Internet Search Object")
If Add.Length >= 1 Then
If Add.Contains(",") Then
Dim split() As String = Add.Split(",")
SetProgressBar(UBound(split), 0)
For I As Integer = 0 To UBound(split)
ProBar.Value = I
If split(I).StartsWith(" ") = True Then split(I) = split(I).Remove(0, 1)
Dim IsIn As Boolean = False
For J As Integer = 0 To InternetSearchObjects.Items.Count - 1
If InternetSearchObjects.Items(J).ToString.Contains(split(I).ToString) Then IsIn = True
Next
If IsIn = False Then InternetSearchObjects.Items.Add(FilterOut(split(I).ToString))
Next
Else
InternetSearchObjects.Items.Add(FilterOut(Add))
End If
End If
SetProgressBar(InternetSearchObjects.Items.Count - 1, 0, 0)
For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
ProBar.Value = I
Dim split() As String = InternetSearchObjects.Items(I).ToString.Split(" ")
For J As Integer = 0 To UBound(split)
Dim IsIn As Boolean = False
For K As Integer = 0 To PageSearchObject.Items.Count - 1
If PageSearchObject.Items(K).ToString.Contains(split(J)) = True Then IsIn = True
Next
If IsIn = False Then PageSearchObject.Items.Add(FilterOut(split(J).ToString))
Next
Next
ProBar.Value = 0
CheckSearchObjects()
End Sub
Private Sub RemoveToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem.Click
Do While (InternetSearchObjects.SelectedItems.Count > 0)
InternetSearchObjects.Items.Remove(InternetSearchObjects.SelectedItem)
Loop
End Sub
' PaSeOb
Private Sub InsertToolStripMenuItem1_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles InsertToolStripMenuItem1.Click
Dim Add As String = InputBox("Object to Search on the Pages" & vbCrLf & "Seperate Items are Created when you use: ,", "Add Page Search Object")
If Add.Contains(",") = True Then
Dim Split() As String = Add.Split(",")
For J As Integer = 0 To UBound(Split)
Dim IsIn As Boolean = False
For K As Integer = 0 To PageSearchObject.Items.Count - 1
If PageSearchObject.Items(K).ToString.Contains(Split(J)) = True Then IsIn = True
Next
If IsIn = False Then PageSearchObject.Items.Add(FilterOut(Split(J).ToString))
Next
Else
Dim IsIn As Boolean = False
For K As Integer = 0 To PageSearchObject.Items.Count - 1
If PageSearchObject.Items(K).ToString.Contains(Add) = True Then IsIn = True
Next
If IsIn = False Then PageSearchObject.Items.Add(FilterOut(Add))
End If
CheckSearchObjects()
End Sub
Private Sub RemoveToolStripMenuItem1_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem1.Click
Do While (PageSearchObject.SelectedItems.Count > 0)
PageSearchObject.Items.Remove(PageSearchObject.SelectedItem)
Loop
End Sub
' FoIn
Private Sub RemoveToolStripMenuItem2_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem2.Click
Do While (FoundInformation.SelectedItems.Count > 0)
FoundInformation.Items.Remove(InternetSearchObjects.SelectedItem)
Loop
End Sub
Private Sub Item_click() Handles FoundInformation.DoubleClick
For I As Integer = 0 To Results.Count - 1
Dim Thing As String = FoundInformation.SelectedItem.ToString
If Thing.Contains(Results(I)) Then
Dim split() As String = Results(I).Split("@ ")
Try
Process.Start(split(1))
Catch
End Try
Exit Sub
End If
Next
End Sub
Private Sub SelectAllToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles SelectAllToolStripMenuItem.Click
For I As Integer = 0 To FoundInformation.Items.Count - 1
FoundInformation.SetSelected(I, True)
Next
End Sub
' Save
Private Sub Save() Handles SaveToolStripMenuItem.Click
SaveFileDialog1.Filter = "Text Files (*.txt)| *.txt| All Files (*.*)| *.*"
SaveFileDialog1.FileName = ""
If SaveFileDialog1.ShowDialog = Windows.Forms.DialogResult.OK Then
Dim Contents As String
Dim Reader As StreamWriter
Contents = "Internet Search Objects:"
For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
Contents += "," & InternetSearchObjects.Items(I).ToString
Next
Contents += "," & vbNewLine & ","
Contents += "Page Search Objects:"
For I As Integer = 0 To PageSearchObject.Items.Count - 1
Contents += "," & PageSearchObject.Items(I).ToString
Next
Contents += "," & vbNewLine & ","
Contents += "Results:"
For I As Integer = 0 To Results.Count - 1
Contents += "," & Results(I).ToString & vbNewLine
Next
Dim Content() As String = Contents.Split(",")
Try
Reader = New StreamWriter(SaveFileDialog1.FileName)
For I As Integer = 0 To UBound(Content)
Reader.WriteLine(Content(I))
Next
Reader.Close()
Catch
End Try
End If
End Sub
' Settings
Private Sub SettingsToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles SettingsToolStripMenuItem.Click
If Settings.ShowDialog = Windows.Forms.DialogResult.OK Then ProText.Text = ""
End Sub
' Results
Private Sub CountItem() Handles ResultsToolStripMenuItem.Click
ItemCountToolStripMenuItem.Text = "Items: " & FoundInformation.Items.Count
End Sub
Private Sub RemoveToolStripMenuItem3_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem3.Click
If FindRemove.ShowDialog = Windows.Forms.DialogResult.OK Then
Dim Search As String = FindRemove.TextBox1.Text
Dim Checked As Boolean = FindRemove.CheckBox1.Checked
Dim Number As Integer = FindRemove.NumericUpDown1.Value
Dim CheckedB As Boolean = FindRemove.CheckBox2.Checked
Dim NumberB As Integer = FindRemove.NumericUpDown2.Value
For I As Integer = 0 To FoundInformation.Items.Count - 1
If I >= FoundInformation.Items.Count - 1 Then Exit For
If FoundInformation.Items(I).ToString.ToLower.Contains(Search.ToLower) Then FoundInformation.Items.RemoveAt(I)
If Number >= FoundInformation.Items(I).ToString.Length And Checked = True Then FoundInformation.Items.RemoveAt(I)
If NumberB <= FoundInformation.Items(I).ToString.Length And CheckedB = True Then FoundInformation.Items.RemoveAt(I)
Next
End If
End Sub
' Search Engine
Function GetListOfSearchEngine() As List(Of String)
Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
If File.Exists(Path) = False Then
MsgBox("Search Engine File is missing")
Application.Exit()
Else
Dim Reader As StreamReader
Dim Content As String = ""
Try
Reader = New StreamReader(Path)
Content = Reader.ReadToEnd
Catch ex As Exception
End Try
Content = Content.Replace(" ", "")
Dim Split() As String = Content.Split(vbNewLine)
Dim Results As New List(Of String)
If UBound(Split) <> 0 Then
For I As Integer = 0 To UBound(Split)
Dim Line As String = Split(I).Replace(" ", "")
If Line.StartsWith("<") = False Then Line = Line.Remove(0, 1)
Line = Line.Replace(vbNewLine, "")
If Line.StartsWith("<") And Line.StartsWith("</") = False And Line.StartsWith("<Filter>") = False _
And Line.StartsWith("</Filter>") = False And Line <> "<>" Then
Results.Add(Line.Replace("<", "").Replace(">", ""))
End If
Next
Return Results
Else
MsgBox("Search Engine File is Corrupted")
Application.Exit()
End If
End If
End Function
Function GetSearchEngineFilterlist(ByVal SearchEngine As String) As List(Of String)
Dim Results As New List(Of String)
Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
If File.Exists(Path) = False Then
MsgBox("Search Engine File is missing")
Application.Exit()
Else
Dim Reader As StreamReader
Dim Content As String = ""
Try
Reader = New StreamReader(Path)
Content = Reader.ReadToEnd
Catch ex As Exception
End Try
Content = Content.Replace(" ", "")
Dim Split() As String = Content.Split(vbNewLine)
For I As Integer = 0 To UBound(Split)
If Split(I).Contains("<" & SearchEngine & ">") = True Then
For J As Integer = I To UBound(Split)
If Split(J).Contains("</Filter>") Then
For K = I + 3 To J - 1
If Split(K).StartsWith("<") = False Then Results.Add(Split(K).Remove(0, 1))
Next
Return Results
Exit Function
End If
Next
End If
Next
End If
End Function
Function GetSearchEngineURL(ByVal SearchEngine As String) As String
Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
If File.Exists(Path) = False Then
MsgBox("Search Engine File is missing")
Application.Exit()
Else
Dim Reader As StreamReader
Dim Content As String = ""
Try
Reader = New StreamReader(Path)
Content = Reader.ReadToEnd
Catch ex As Exception
End Try
Content = Content.Replace(" ", "")
Dim Split() As String = Content.Split(vbNewLine)
For I As Integer = 0 To UBound(Split)
If Split(I).Contains("<" & SearchEngine & ">") = True Then
Dim Res As String = Split(I + 1).Remove(0, 5).Replace(" ", "").Replace(vbNewLine, "")
Return Res
Exit Function
End If
Next
End If
End Function
'Functions
Function ResetStuff()
ProBar.Value = 0
ProcesStop = False
ProText.Text = ""
StartToolStripMenuItem.Enabled = True
End Function
Function FilterUrls(ByVal Urls() As String) As String()
Dim CheckedUrls As String = ""
If SearchEngineFilterlist.Count >= 0 Then
For I As Integer = 0 To UBound(Urls)
For J As Integer = 0 To SearchEngineFilterlist.Count - 1
Dim Isin As Boolean = False
If Urls(I).Contains(SearchEngineFilterlist(J)) Then Isin = True
If Isin = False Then CheckedUrls += Urls(I) & ","
Next
Next
End If
Return CheckedUrls.Split(",")
End Function
Function GetUrls(ByVal Content As String) As String()
Dim urls As String
Dim Quote As String = Convert.ToChar(34)
For X As Integer = 1 To Content.Length
Dim Search As String = Mid(Content, X, 6)
If Search = "href=" & Quote Then
Dim Xb As Integer = 7
Do While (Content(X + Xb) <> Quote)
Xb += 1
Loop
urls += Mid(Content, X, Xb + 1) & ","
X += Xb
End If
Next
If urls <> "" Then
urls = urls.Replace("href=" & Quote, "")
urls = urls.Replace(Quote, "")
If urls.EndsWith(",") = True Then urls = urls.Remove(urls.Length - 1, 1)
Return urls.Split(",")
End If
End Function
Function GetPage(ByVal Url As String) As String
Dim Browser As New Net.WebClient
Dim Bytes() As Byte
Dim Text As String = ""
Try
Bytes = Browser.DownloadData(Url)
Text = System.Text.ASCIIEncoding.ASCII.GetString(Bytes)
Browser.Dispose()
Catch
End Try
Return Text
End Function
Function GetSearchEnginePage(ByVal SearchObject As String, ByVal PageNumber As Integer) As String
Dim Browser As New Net.WebClient
Dim Url As String = (SearchEngineURL.Replace("SEARCHOBJECT", (SearchObject.Replace(" ", "+"))).Replace("PAGE", PageNumber & "0")).Replace(" ", "")
Dim Text As String = ""
Dim Bytes() As Byte
If Url.StartsWith("http://") = False Then Url = "http://" & Url
Bytes = Browser.DownloadData(Url.ToString.ToLower)
Text = System.Text.ASCIIEncoding.ASCII.GetString(Bytes)
Browser.Dispose()
Return Text
End Function
Function SetProgressBar(ByVal Max As Integer, ByVal Min As Integer, Optional ByVal Value As Integer = 0)
If Max <= 0 Then Max = 1
If Min <= 0 Then Min = 0
ProBar.Maximum = Max
ProBar.Minimum = Min
End Function
Function UpdateStuff(Optional ByVal ProgressBar As Integer = 0, Optional ByVal ProgressText As String = "", Optional ByVal FormText As String = "Advanced Internet Search")
ProBar.Value = ProgressBar
ProText.Text = ProgressText
Console.Writeline(ProgressText)
Me.Text = FormText
ProBar.Update()
ProText.Update()
Me.Update()
MenuStrip1.Update()
End Function
Function GetString(ByVal SearchObject As String, ByVal Contents As String) As List(Of String)
Dim Results As New List(Of String)
For Index As Integer = 1 To Contents.Length
Dim Word As String = Mid(Contents, Index, SearchObject.Length)
If Word = SearchObject Then
Dim startpoint As Integer = Index
Dim Chars As Char = Mid(Contents, startpoint, 1)
Dim Quote As String = Convert.ToChar(34)
Do Until (Chars = ":" Or Chars = "." Or Chars = "," Or Chars = Quote Or Chars = ">" _
Or Chars = "<" Or Chars = "(" Or Chars = ")" Or Chars = vbNewLine Or Chars = Environment.NewLine Or Chars = vbCrLf)
startpoint -= 1
If startpoint = 0 Then Exit Do
Chars = Mid(Contents, startpoint, 1)
Loop
Dim Endpoint As Integer = Index
Chars = Mid(Contents, Endpoint, 1)
Do Until (Chars = ":" Or Chars = "." Or Chars = "," Or Chars = Quote Or Chars = ">" _
Or Chars = "<" Or Chars = "(" Or Chars = ")")
Endpoint += 1
If Endpoint > Contents.Length Then Exit Do
Chars = Mid(Contents, Endpoint, 1)
Loop
Results.Add(Mid(Contents, startpoint + 1, (Endpoint - startpoint)))
End If
Next
Return Results
End Function
Function DetectUrl(ByVal Tekst As String, ByVal Search As String) As Boolean
Dim Bool As Boolean = False
If (Tekst.Contains("/") Or Tekst.Contains("%") Or Tekst.Contains("?")) And Not (Tekst.Contains(" ")) _
And Tekst.Length > (1.5 * Search.Length) Then Bool = True
Return Bool
End Function
Function GetFilterList() As List(Of String)
Dim ByteStream As FileStream
Dim CurrentByte As Integer
Dim lngLoop As Long = 0
Dim ByteContent() As Byte
Dim Contents As String
Dim FinalList As New List(Of String)
If File.Exists("FilterLibrary.ais") = False Then
FinalList.Add("")
Return FinalList
Exit Function
End If
Try
ByteStream = File.OpenRead("FilterLibrary.ais")
ReDim ByteContent(ByteStream.Length)
Do While Not CurrentByte = -1
CurrentByte = ByteStream.ReadByte()
If CurrentByte <> -1 Then ByteContent(lngLoop) = CByte(CurrentByte)
lngLoop += 1
Loop
Contents = Encoding.ASCII.GetString(ByteContent)
ByteStream.Close()
Catch ex As IOException
End Try
Contents = Contents.ToLower
Dim Split() As String = Contents.Split("#")
For I As Integer = 0 To UBound(Split)
FinalList.Add(Split(I))
Next
Return FinalList
End Function
Function FilterOut(ByVal Input As String) As String
For I As Integer = 0 To FilterOutList.Count - 1
Input = Input.Replace(FilterOutList(I), "")
Next
Return Input
End Function
Function CheckSearchObjects()
For I As Integer = 0 To PageSearchObject.Items.Count - 1
If PageSearchObject.Items(I).ToString.Length = 0 Then PageSearchObject.Items.RemoveAt(I)
If I >= PageSearchObject.Items.Count Then Exit For
Next
For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
If InternetSearchObjects.Items(I).ToString.Length = 0 Then InternetSearchObjects.Items.RemoveAt(I)
If I >= InternetSearchObjects.Items.Count Then Exit For
Next
End Function
End Class
THE WHOLE CREDIT GOES TO THE ORIGINAL PROGRAMMER --- Huntondoom
0 comments:
Post a Comment