Page 1 of 1

Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 8:59 am
by pieter
Dear member,

I am trying to compare the speed of dbf(cdx/ntx/ads) with adordd. I want to improve the speed of adordd, but before that I should know what the current situation is exactly. Therefor I made a little testapp, especially to compare the DbSeek function for dbf and adordd.

Is there a way to count the duration more precisely? Now sometimes it gives 0.00 (with dbf).

I am using something like this:

Code: Select all

nsecs := Seconds()
USE LargeDbf
duration1 := Seconds() - nsec
memowrit("log.txt", Memoread("log.txt") + "|" + "USE LargeDbf: " + str(duration1)
 
I have also tested MsgInfo( 5 / 0 ) which gives 0. I think this should give an error.

Code: Select all

#include "Adordd.ch"
#include "adodef.ch"

static oCn

PROCEDURE Main()

//createLargeDbfFile()
//ConvertDbf()

array1Dbf := RunDbSeekTest(.F.,10000)
array2Ado := RunDbSeekTest(.T.,10000)

DbfTimesFasterThanAdorddUse := array2Ado[1] / array1Dbf[1]
DbfTimesFasterThanAdorddIndex := array2Ado[2] / array1Dbf[2]
DbfTimesFasterThanAdorddDBSEEK := array2Ado[3] / array1Dbf[3]
DbfTimesFasterThanAdorddAVDBSEEK := array2Ado[4] / array1Dbf[4]
 
memowrit("log.txt",MemoRead("log.txt") + " " + "DbfTimesFasterThanAdorddUse: " + str(DbfTimesFasterThanAdorddUse) + " " + "DbfTimesFasterThanAdorddIndex: " + str(DbfTimesFasterThanAdorddIndex) + " " + "DbfTimesFasterThanAdorddDBSEEK: " + str(DbfTimesFasterThanAdorddDBSEEK) + " " + "DbfTimesFasterThanAdorddAVDBSEEK: " + str(DbfTimesFasterThanAdorddAVDBSEEK) )

RETURN

FUNCTION RunDbSeekTest(bSQL, numberOfLoop)

memowrit("log.txt",MemoRead("log.txt") + "Begin DbSeek=" + "SQL: " + IF(bSQL, "TRUE", "FALSE") + "|numberOfLoop: " + str(numberOfLoop) + " ")

IF bSQL

//MsgInfo( ADOVERSION() )
******
REQUEST ADORDD
RddRegister("ADORDD",1)
RddSetDefault("ADORDD")
SET ADO FORCE LOCK OFF   // Required!
SET ADO DEFAULT DATABASE TO "dbseekdb" SERVER TO "localhost"  ENGINE TO "MYSQL" USER TO "pieter" PASSWORD TO "password"
SET ADO TEMPORAY NAMES INDEX LIST TO {"TMP","TEMP"}
SET ADO DEFAULT RECNO FIELD TO "HBRECNO"
SET ADO DEFAULT DELETED FIELD TO "HBDELETED"
SET ADODBF TABLES INDEX LIST TO { {"LARGEDBF", {"CO1", "COLUMN1"}, {"CO2", "COLUMN2"}}}
SET AUTOPEN ON

ELSE

REQUEST DBFCDX
RddRegister("DBFCDX",1)
RddSetDefault("DBFCDX")

ENDIF

nsecs := Seconds()
USE LargeDbf
duration1 := Seconds() - nsecs
memowrit("log.txt", Memoread("log.txt") + "|" + "USE LargeDbf: " + str(duration1))

nsecs := Seconds()
INDEX ON COLUMN1 TO LargeDbf
duration2 := Seconds() - nsecs
memowrit("log.txt", Memoread("log.txt") + "|" + "INDEX ON COLUMN1 TO LargeDbf: " +  str(duration2))

nsecs := Seconds()
FOR i:= 1 TO numberOfLoop  
DBSEEK(numberOfLoop / 2) 
NEXT
duration3 := Seconds() - nsecs
memowrit("log.txt", Memoread("log.txt") + "|" + "FOR DBSEEK Duration: " + str(duration3))
AverageDuration := duration3 / numberOfLoop 
memowrit("log.txt", Memoread("log.txt") + "|" + "Average DBSEEK Duration: " + str(AverageDuration))

RETURN {duration1, duration2, duration3, AverageDuration }

FUNCTION createLargeDbfFile()

Create StructureLargeDbf

APPEND BLANK
FIELD->FIELD_NAME := "Column1"
FIELD->FIELD_TYPE := "N"
FIELD->FIELD_LEN  := 8
FIELD->FIELD_DEC  := 0

FOR i:=2 TO 20
    APPEND BLANK
    ts := ALLTRIM(STR(i,8))
    FIELD->FIELD_NAME := "Column" + ts
    FIELD->FIELD_TYPE := "C"
    FIELD->FIELD_LEN  := 20
    FIELD->FIELD_DEC  := 0
NEXT

CLOSE

CREATE LargeDbf FROM StructureLargeDbf

FOR j:=1 TO 20000
    APPEND BLANK
    FOR x := 1 TO 20
        
        IF x = 1 
            REPLACE Column1 WITH j
        ELSE
            ts := ALLTRIM(STR(x,3))
            ts2 := ALLTRIM(STR(HB_RandomInt(1000),5))
            ts3 := "Column" + ts
            ts4 := "td" + ts2
            REPLACE &ts3 WITH ts4
        ENDIF
    NEXT
NEXT

CLOSE

RETURN NIL

function ConvertDbf() 
    CreateDatabaseIfNotYetExist( "dbseekdb" )
    ConnectWithDatabase( "dbseekdb" )  
    FW_AdoImportFromDBF( oCn, "C:\Pieter\LeerOmgevingHarbour\DbSeek\largedbf.dbf", "largedbf")
RETURN

//----------------------------------------------------------------------------//

FUNCTION CloseDatabase()
   oCn:Close()
RETURN NIL   

FUNCTION ConnectWithDatabase( vardb )
   ADOCONNECT oCn TO MYSQL SERVER localhost DATABASE &vardb USER pieter PASSWORD "password" // PASSWORD ... 
      
   if oCn == nil .or. oCn:State < 1
      MsgInfo( "Connect failed" )
      return nil
   endif         
 
   //MsgInfo( "Connection Open" )
RETURN NIL 
   
FUNCTION CreateDatabaseIfNotYetExist( vardb )
   local oError
   
   ADOCONNECT oCn TO MYSQL SERVER localhost USER pieter PASSWORD password
 
   if oCn == nil
      MsgInfo( "Not connected" )
    else
      if oCn:State > 0
         //MsgInfo( "open" )
         
      TRY
         oCn:Execute( "CREATE DATABASE " + vardb )
         //MsgInfo( "created" )      
      CATCH oError
         MsgInfo( "The database already exists" )
      END          
         
      else
         MsgInfo( "not open" )
      endif
   endif         
 
   oCn:Close() 
  
Return nil
 
Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 9:18 am
by Antonio Linares
Pieter,

Use GetTickCount() instead of Seconds()

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 9:22 am
by pieter
Antonio,

Thanks, I will use GetTickCount().

Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 10:24 am
by pieter
Hi,

Results:
Number of loops 10000:
DbSeek(5000):
according to my test Dbf is more than 3000 times faster than adordd for DbSeek(5000)

Code: Select all

nCount := GetTickCount()
FOR i:= 1 TO numberOfLoop  //10000
DBSEEK(numberOfLoop / 2) //5000
NEXT
duration3 := GetTickCount() - nCount
memowrit("log.txt", Memoread("log.txt") + "|" + "FOR DBSEEK Duration: " + str(duration3)
 

Code: Select all

DbfTimesFasterThanAdorddDBSEEK := array2Ado[3] / array1Dbf[3] //for three times testing it gives: 3388.69, 3462.88 ,3337.88
 
I want to make my results overview a bit better. I have now everthing on one line in log.txt

Question:
How can I add a new line in memowrit("log.txt", Memoread("log.txt") + "|" + "FOR DBSEEK Duration: " + str(duration3)? (I thought something with \n)

Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 10:30 am
by Antonio Linares
Pieter,

#include "FiveWin.ch"

... + CRLF + ...

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 10:42 am
by pieter
Antonio, Great!, I got a better results overview now.

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 11:37 am
by Enrico Maria Giordano
pieter wrote:according to my test Dbf is more than 3000 times faster than adordd for DbSeek(5000)
Is your DBF test over a LAN?

EMG

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 11:51 am
by pieter
Hi Enrico,

"Is your DBF test over a LAN?"

No, I use everthing locally on my operating system, Dbf and mysql server.

I am curious whether other people have better speed results for adordd. I think that dbf is very fast on local operating system compared with adordd. (without LAN etc.)

Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 11:56 am
by Enrico Maria Giordano
pieter wrote:No, I use everthing locally on my operating system, Dbf and mysql server.
So your test is not meaningful. You have to make both test over a network to get significant results.

EMG

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 12:09 pm
by pieter
Enrico,

My companies main app is actually a kind of desktop app. With a remote desktop tool one can acces the app from a distance. So I think the main app does not use something like sending data via LAN.

"So your test is not meaningful. You have to make both test over a network to get significant results."

Can you explain why both tests have to be over a network to get significant results?

Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 12:29 pm
by Enrico Maria Giordano
pieter wrote:Can you explain why both tests have to be over a network to get significant results?
Because SQL has the client/server overhead even if installed locally.

EMG

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 12:56 pm
by pieter
Enrico, Thanks.

"Because SQL has the client/server overhead even if installed locally".

Do you mean with "SQL" a sql server such mysql or ado or adordd?

Pieter

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 1:00 pm
by Enrico Maria Giordano
pieter wrote:Do you mean with "SQL" a sql server such mysql or ado or adordd?
Yes, a SQL server. ADO and ADORDD are just interface to a SQL server.

EMG

Re: Little speed test app dbf and adordd

Posted: Mon Mar 21, 2016 1:34 pm
by pieter
Oké!, I understand.

How can I do something with a network so that I can have a good comparison between dbf and sql/adordd?

Results:
Begin DbSeek=SQL: FALSE numberOfLoop: 10000
USE LargeDbf: 0
INDEX ON COLUMN1 TO LargeDbf: 15
FOR DBSEEK Duration: 16
Average DBSEEK Duration: 0.00

Begin DbSeek=SQL: TRUE numberOfLoop: 10000
USE LargeDbf: 8344
INDEX ON COLUMN1 TO LargeDbf: 3422
FOR DBSEEK Duration: 62469
Average DBSEEK Duration: 6.25

DbfTimesFasterThanAdorddUse: 0
DbfTimesFasterThanAdorddIndex: 228.13
DbfTimesFasterThanAdorddDBSEEK: 3904.31
DbfTimesFasterThanAdorddAVDBSEEK: 3904.31

Code: Select all

#include "Adordd.ch"
#include "adodef.ch"
#include "fivewin.ch"

static oCn

PROCEDURE Main()

//createLargeDbfFile()
//ConvertDbf()

array1Dbf := RunDbSeekTest(.F.,10000)
array2Ado := RunDbSeekTest(.T.,10000)

DbfTimesFasterThanAdorddUse := array2Ado[1] / array1Dbf[1]
DbfTimesFasterThanAdorddIndex := array2Ado[2] / array1Dbf[2]
DbfTimesFasterThanAdorddDBSEEK := array2Ado[3] / array1Dbf[3]
DbfTimesFasterThanAdorddAVDBSEEK := array2Ado[4] / array1Dbf[4]
 
memowrit("log.txt",MemoRead("log.txt") + "DbfTimesFasterThanAdorddUse: " + str(DbfTimesFasterThanAdorddUse) +  CRLF  + "DbfTimesFasterThanAdorddIndex: " + str(DbfTimesFasterThanAdorddIndex) +  CRLF  + "DbfTimesFasterThanAdorddDBSEEK: " + str(DbfTimesFasterThanAdorddDBSEEK) + CRLF + "DbfTimesFasterThanAdorddAVDBSEEK: " + str(DbfTimesFasterThanAdorddAVDBSEEK) + CRLF + CRLF + CRLF)

RETURN

FUNCTION RunDbSeekTest(bSQL, numberOfLoop)

memowrit("log.txt",MemoRead("log.txt") + "Begin DbSeek=" + "SQL: " + IF(bSQL, "TRUE", "FALSE") + " " + "numberOfLoop: " + str(numberOfLoop) + CRLF )

IF bSQL

//MsgInfo( ADOVERSION() )
******
REQUEST ADORDD
RddRegister("ADORDD",1)
RddSetDefault("ADORDD")
SET ADO FORCE LOCK OFF   // Required!
SET ADO DEFAULT DATABASE TO "dbseekdb" SERVER TO "localhost"  ENGINE TO "MYSQL" USER TO "pieter" PASSWORD TO "password"
SET ADO TEMPORAY NAMES INDEX LIST TO {"TMP","TEMP"}
SET ADO DEFAULT RECNO FIELD TO "HBRECNO"
SET ADO DEFAULT DELETED FIELD TO "HBDELETED"
SET ADODBF TABLES INDEX LIST TO { {"LARGEDBF", {"CO1", "COLUMN1"}, {"CO2", "COLUMN2"}}}
SET AUTOPEN ON

ELSE

REQUEST DBFCDX
RddRegister("DBFCDX",1)
RddSetDefault("DBFCDX")

ENDIF

nCount := GetTickCount()
USE LargeDbf
duration1 := GetTickCount() - nCount
memowrit("log.txt", Memoread("log.txt") + "USE LargeDbf: " + str(duration1) + CRLF )

nCount := GetTickCount()
INDEX ON COLUMN1 TO LargeDbf
duration2 := GetTickCount() - nCount
memowrit("log.txt", Memoread("log.txt")  + "INDEX ON COLUMN1 TO LargeDbf: " +  str(duration2) + CRLF)

nCount := GetTickCount()
FOR i:= 1 TO numberOfLoop  
DBSEEK(numberOfLoop / 2) 
NEXT
duration3 := GetTickCount() - nCount
memowrit("log.txt", Memoread("log.txt") + "FOR DBSEEK Duration: " + str(duration3) + CRLF )
AverageDuration := duration3 / numberOfLoop 
memowrit("log.txt", Memoread("log.txt") + "Average DBSEEK Duration: " + str(AverageDuration) + CRLF + CRLF)

RETURN {duration1, duration2, duration3, AverageDuration }

FUNCTION createLargeDbfFile()

Create StructureLargeDbf

APPEND BLANK
FIELD->FIELD_NAME := "Column1"
FIELD->FIELD_TYPE := "N"
FIELD->FIELD_LEN  := 8
FIELD->FIELD_DEC  := 0

FOR i:=2 TO 20
    APPEND BLANK
    ts := ALLTRIM(STR(i,8))
    FIELD->FIELD_NAME := "Column" + ts
    FIELD->FIELD_TYPE := "C"
    FIELD->FIELD_LEN  := 20
    FIELD->FIELD_DEC  := 0
NEXT

CLOSE

CREATE LargeDbf FROM StructureLargeDbf

FOR j:=1 TO 20000
    APPEND BLANK
    FOR x := 1 TO 20
        
        IF x = 1 
            REPLACE Column1 WITH j
        ELSE
            ts := ALLTRIM(STR(x,3))
            ts2 := ALLTRIM(STR(HB_RandomInt(1000),5))
            ts3 := "Column" + ts
            ts4 := "td" + ts2
            REPLACE &ts3 WITH ts4
        ENDIF
    NEXT
NEXT

CLOSE

RETURN NIL

function ConvertDbf() 
    CreateDatabaseIfNotYetExist( "dbseekdb2" )
    ConnectWithDatabase( "dbseekdb2" )  
    FW_AdoImportFromDBF( oCn, "C:\Pieter\LeerOmgevingHarbour\DbSeek\largedbf.dbf", "largedbf")
RETURN

//----------------------------------------------------------------------------//

FUNCTION CloseDatabase()
   oCn:Close()
RETURN NIL   

FUNCTION ConnectWithDatabase( vardb )
   ADOCONNECT oCn TO MYSQL SERVER localhost DATABASE &vardb USER pieter PASSWORD "password" // PASSWORD ... 
      
   if oCn == nil .or. oCn:State < 1
      MsgInfo( "Connect failed" )
      return nil
   endif         
 
   //MsgInfo( "Connection Open" )
RETURN NIL 
   
FUNCTION CreateDatabaseIfNotYetExist( vardb )
   local oError
   
   ADOCONNECT oCn TO MYSQL SERVER localhost USER pieter PASSWORD password
 
   if oCn == nil
      MsgInfo( "Not connected" )
    else
      if oCn:State > 0
         //MsgInfo( "open" )
         
      TRY
         oCn:Execute( "CREATE DATABASE " + vardb )
         //MsgInfo( "created" )      
      CATCH oError
         MsgInfo( "The database already exists" )
      END          
         
      else
         MsgInfo( "not open" )
      endif
   endif         
 
   oCn:Close() 
  
Return nil
Pieter